libavformat/cache.c

   1 /*
   2  * Input cache protocol.
   3  * Copyright (c) 2011,2014 Michael Niedermayer
   4  *
   5  * This file is part of FFmpeg.
   6  *
   7  * FFmpeg is free software; you can redistribute it and/or
   8  * modify it under the terms of the GNU Lesser General Public
   9  * License as published by the Free Software Foundation; either
  10  * version 2.1 of the License, or (at your option) any later version.
  11  *
  12  * FFmpeg is distributed in the hope that it will be useful,
  13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  15  * Lesser General Public License for more details.
  16  *
  17  * You should have received a copy of the GNU Lesser General Public
  18  * License along with FFmpeg; if not, write to the Free Software
  19  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
  20  *
  21  * Based on file.c by Fabrice Bellard
  22  */
  23
  24 /**
  25  * @TODO
  26  *      support keeping files
  27  *      support filling with a background thread
  28  */
  29
  30 #include "libavutil/avassert.h"
  31 #include "libavutil/avstring.h"
  32 #include "libavutil/internal.h"
  33 #include "libavutil/opt.h"
  34 #include "libavutil/tree.h"
  35 #include "avformat.h"
  36 #include <fcntl.h>
  37 #if HAVE_IO_H
  38 #include <io.h>
  39 #endif
  40 #if HAVE_UNISTD_H
  41 #include <unistd.h>
  42 #endif
  43 #include <sys/stat.h>
  44 #include <stdlib.h>
  45 #include "os_support.h"
  46 #include "url.h"
  47
  48 typedef struct CacheEntry {
  49     int64_t logical_pos;
  50     int64_t physical_pos;
  51     int size;
  52 } CacheEntry;
  53
  54 typedef struct Context {
  55     AVClass *class;
  56     int fd;
  57     char *filename;
  58     struct AVTreeNode *root;
  59     int64_t logical_pos;
  60     int64_t cache_pos;
  61     int64_t inner_pos;
  62     int64_t end;
  63     int is_true_eof;
  64     URLContext *inner;
  65     int64_t cache_hit, cache_miss;
  66     int read_ahead_limit;
  67 } Context;
  68
  69 static int cmp(const void *key, const void *node)
  70 {
  71     return FFDIFFSIGN(*(const int64_t *)key, ((const CacheEntry *) node)->logical_pos);
  72 }
  73
  74 static int cache_open(URLContext *h, const char *arg, int flags, AVDictionary **options)
  75 {
  76     int ret;
  77     char *buffername;
  78     Context *c= h->priv_data;
  79
  80     av_strstart(arg, "cache:", &arg);
  81
  82     c->fd = avpriv_tempfile("ffcache", &buffername, 0, h);
  83     if (c->fd < 0){
  84         av_log(h, AV_LOG_ERROR, "Failed to create tempfile\n");
  85         return c->fd;
  86     }
  87
  88     ret = unlink(buffername);
  89
  90     if (ret >= 0)
  91         av_freep(&buffername);
  92     else
  93         c->filename = buffername;
  94
  95     return ffurl_open_whitelist(&c->inner, arg, flags, &h->interrupt_callback,
  96                                 options, h->protocol_whitelist, h->protocol_blacklist, h);
  97 }
  98
  99 static int add_entry(URLContext *h, const unsigned char *buf, int size)
 100 {
 101     Context *c= h->priv_data;
 102     int64_t pos = -1;
 103     int ret;
 104     CacheEntry *entry = NULL, *next[2] = {NULL, NULL};
 105     CacheEntry *entry_ret;
 106     struct AVTreeNode *node = NULL;
 107
 108     //FIXME avoid lseek
 109     pos = lseek(c->fd, 0, SEEK_END);
 110     if (pos < 0) {
 111         ret = AVERROR(errno);
 112         av_log(h, AV_LOG_ERROR, "seek in cache failed\n");
 113         goto fail;
 114     }
 115     c->cache_pos = pos;
 116
 117     ret = write(c->fd, buf, size);
 118     if (ret < 0) {
 119         ret = AVERROR(errno);
 120         av_log(h, AV_LOG_ERROR, "write in cache failed\n");
 121         goto fail;
 122     }
 123     c->cache_pos += ret;
 124
 125     entry = av_tree_find(c->root, &c->logical_pos, cmp, (void**)next);
 126
 127     if (!entry)
 128         entry = next[0];
 129
 130     if (!entry ||
 131         entry->logical_pos  + entry->size != c->logical_pos ||
 132         entry->physical_pos + entry->size != pos
 133     ) {
 134         entry = av_malloc(sizeof(*entry));
 135         node = av_tree_node_alloc();
 136         if (!entry || !node) {
 137             ret = AVERROR(ENOMEM);
 138             goto fail;
 139         }
 140         entry->logical_pos = c->logical_pos;
 141         entry->physical_pos = pos;
 142         entry->size = ret;
 143
 144         entry_ret = av_tree_insert(&c->root, entry, cmp, &node);
 145         if (entry_ret && entry_ret != entry) {
 146             ret = -1;
 147             av_log(h, AV_LOG_ERROR, "av_tree_insert failed\n");
 148             goto fail;
 149         }
 150     } else
 151         entry->size += ret;
 152
 153     return 0;
 154 fail:
 155     //we could truncate the file to pos here if pos >=0 but ftruncate isn't available in VS so
 156     //for simplicty we just leave the file a bit larger
 157     av_free(entry);
 158     av_free(node);
 159     return ret;
 160 }
 161
 162 static int cache_read(URLContext *h, unsigned char *buf, int size)
 163 {
 164     Context *c= h->priv_data;
 165     CacheEntry *entry, *next[2] = {NULL, NULL};
 166     int64_t r;
 167
 168     entry = av_tree_find(c->root, &c->logical_pos, cmp, (void**)next);
 169
 170     if (!entry)
 171         entry = next[0];
 172
 173     if (entry) {
 174         int64_t in_block_pos = c->logical_pos - entry->logical_pos;
 175         av_assert0(entry->logical_pos <= c->logical_pos);
 176         if (in_block_pos < entry->size) {
 177             int64_t physical_target = entry->physical_pos + in_block_pos;
 178
 179             if (c->cache_pos != physical_target) {
 180                 r = lseek(c->fd, physical_target, SEEK_SET);
 181             } else
 182                 r = c->cache_pos;
 183
 184             if (r >= 0) {
 185                 c->cache_pos = r;
 186                 r = read(c->fd, buf, FFMIN(size, entry->size - in_block_pos));
 187             }
 188
 189             if (r > 0) {
 190                 c->cache_pos += r;
 191                 c->logical_pos += r;
 192                 c->cache_hit ++;
 193                 return r;
 194             }
 195         }
 196     }
 197
 198     // Cache miss or some kind of fault with the cache
 199
 200     if (c->logical_pos != c->inner_pos) {
 201         r = ffurl_seek(c->inner, c->logical_pos, SEEK_SET);
 202         if (r<0) {
 203             av_log(h, AV_LOG_ERROR, "Failed to perform internal seek\n");
 204             return r;
 205         }
 206         c->inner_pos = r;
 207     }
 208
 209     r = ffurl_read(c->inner, buf, size);
 210     if (r == AVERROR_EOF && size>0) {
 211         c->is_true_eof = 1;
 212         av_assert0(c->end >= c->logical_pos);
 213     }
 214     if (r<=0)
 215         return r;
 216     c->inner_pos += r;
 217
 218     c->cache_miss ++;
 219
 220     add_entry(h, buf, r);
 221     c->logical_pos += r;
 222     c->end = FFMAX(c->end, c->logical_pos);
 223
 224     return r;
 225 }
 226
 227 static int64_t cache_seek(URLContext *h, int64_t pos, int whence)
 228 {
 229     Context *c= h->priv_data;
 230     int64_t ret;
 231
 232     if (whence == AVSEEK_SIZE) {
 233         pos= ffurl_seek(c->inner, pos, whence);
 234         if(pos <= 0){
 235             pos= ffurl_seek(c->inner, -1, SEEK_END);
 236             if (ffurl_seek(c->inner, c->inner_pos, SEEK_SET) < 0)
 237                 av_log(h, AV_LOG_ERROR, "Inner protocol failed to seekback end : %"PRId64"\n", pos);
 238         }
 239         if (pos > 0)
 240             c->is_true_eof = 1;
 241         c->end = FFMAX(c->end, pos);
 242         return pos;
 243     }
 244
 245     if (whence == SEEK_CUR) {
 246         whence = SEEK_SET;
 247         pos += c->logical_pos;
 248     } else if (whence == SEEK_END && c->is_true_eof) {
 249 resolve_eof:
 250         whence = SEEK_SET;
 251         pos += c->end;
 252     }
 253
 254     if (whence == SEEK_SET && pos >= 0 && pos < c->end) {
 255         //Seems within filesize, assume it will not fail.
 256         c->logical_pos = pos;
 257         return pos;
 258     }
 259
 260     //cache miss
 261     ret= ffurl_seek(c->inner, pos, whence);
 262     if ((whence == SEEK_SET && pos >= c->logical_pos ||
 263          whence == SEEK_END && pos <= 0) && ret < 0) {
 264         if (   (whence == SEEK_SET && c->read_ahead_limit >= pos - c->logical_pos)
 265             || c->read_ahead_limit < 0) {
 266             uint8_t tmp[32768];
 267             while (c->logical_pos < pos || whence == SEEK_END) {
 268                 int size = sizeof(tmp);
 269                 if (whence == SEEK_SET)
 270                     size = FFMIN(sizeof(tmp), pos - c->logical_pos);
 271                 ret = cache_read(h, tmp, size);
 272                 if (ret == AVERROR_EOF && whence == SEEK_END) {
 273                     av_assert0(c->is_true_eof);
 274                     goto resolve_eof;
 275                 }
 276                 if (ret < 0) {
 277                     return ret;
 278                 }
 279             }
 280             return c->logical_pos;
 281         }
 282     }
 283
 284     if (ret >= 0) {
 285         c->logical_pos = ret;
 286         c->end = FFMAX(c->end, ret);
 287     }
 288
 289     return ret;
 290 }
 291
 292 static int enu_free(void *opaque, void *elem)
 293 {
 294     av_free(elem);
 295     return 0;
 296 }
 297
 298 static int cache_close(URLContext *h)
 299 {
 300     Context *c= h->priv_data;
 301     int ret;
 302
 303     av_log(h, AV_LOG_INFO, "Statistics, cache hits:%"PRId64" cache misses:%"PRId64"\n",
 304            c->cache_hit, c->cache_miss);
 305
 306     close(c->fd);
 307     if (c->filename) {
 308         ret = unlink(c->filename);
 309         if (ret < 0)
 310             av_log(h, AV_LOG_ERROR, "Could not delete %s.\n", c->filename);
 311         av_freep(&c->filename);
 312     }
 313     ffurl_closep(&c->inner);
 314     av_tree_enumerate(c->root, NULL, NULL, enu_free);
 315     av_tree_destroy(c->root);
 316
 317     return 0;
 318 }
 319
 320 #define OFFSET(x) offsetof(Context, x)
 321 #define D AV_OPT_FLAG_DECODING_PARAM
 322
 323 static const AVOption options[] = {
 324     { "read_ahead_limit", "Amount in bytes that may be read ahead when seeking isn't supported, -1 for unlimited", OFFSET(read_ahead_limit), AV_OPT_TYPE_INT, { .i64 = 65536 }, -1, INT_MAX, D },
 325     {NULL},
 326 };
 327
 328 static const AVClass cache_context_class = {
 329     .class_name = "cache",
 330     .item_name  = av_default_item_name,
 331     .option     = options,
 332     .version    = LIBAVUTIL_VERSION_INT,
 333 };
 334
 335 const URLProtocol ff_cache_protocol = {
 336     .name                = "cache",
 337     .url_open2           = cache_open,
 338     .url_read            = cache_read,
 339     .url_seek            = cache_seek,
 340     .url_close           = cache_close,
 341     .priv_data_size      = sizeof(Context),
 342     .priv_data_class     = &cache_context_class,
 343 };