libavformat/cache.c

   1 /*
   2  * Input cache protocol.
   3  * Copyright (c) 2011,2014 Michael Niedermayer
   4  *
   5  * This file is part of FFmpeg.
   6  *
   7  * FFmpeg is free software; you can redistribute it and/or
   8  * modify it under the terms of the GNU Lesser General Public
   9  * License as published by the Free Software Foundation; either
  10  * version 2.1 of the License, or (at your option) any later version.
  11  *
  12  * FFmpeg is distributed in the hope that it will be useful,
  13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  15  * Lesser General Public License for more details.
  16  *
  17  * You should have received a copy of the GNU Lesser General Public
  18  * License along with FFmpeg; if not, write to the Free Software
  19  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
  20  *
  21  * Based on file.c by Fabrice Bellard
  22  */
  23
  24 /**
  25  * @TODO
  26  *      support keeping files
  27  *      support filling with a background thread
  28  */
  29
  30 #include "libavutil/avassert.h"
  31 #include "libavutil/avstring.h"
  32 #include "libavutil/internal.h"
  33 #include "libavutil/opt.h"
  34 #include "libavutil/tree.h"
  35 #include "avformat.h"
  36 #include <fcntl.h>
  37 #if HAVE_IO_H
  38 #include <io.h>
  39 #endif
  40 #if HAVE_UNISTD_H
  41 #include <unistd.h>
  42 #endif
  43 #include <sys/stat.h>
  44 #include <stdlib.h>
  45 #include "os_support.h"
  46 #include "url.h"
  47
  48 typedef struct CacheEntry {
  49     int64_t logical_pos;
  50     int64_t physical_pos;
  51     int size;
  52 } CacheEntry;
  53
  54 typedef struct Context {
  55     AVClass *class;
  56     int fd;
  57     struct AVTreeNode *root;
  58     int64_t logical_pos;
  59     int64_t cache_pos;
  60     int64_t inner_pos;
  61     int64_t end;
  62     int is_true_eof;
  63     URLContext *inner;
  64     int64_t cache_hit, cache_miss;
  65     int read_ahead_limit;
  66 } Context;
  67
  68 static int cmp(const void *key, const void *node)
  69 {
  70     return FFDIFFSIGN(*(const int64_t *)key, ((const CacheEntry *) node)->logical_pos);
  71 }
  72
  73 static int cache_open(URLContext *h, const char *arg, int flags, AVDictionary **options)
  74 {
  75     char *buffername;
  76     Context *c= h->priv_data;
  77
  78     av_strstart(arg, "cache:", &arg);
  79
  80     c->fd = avpriv_tempfile("ffcache", &buffername, 0, h);
  81     if (c->fd < 0){
  82         av_log(h, AV_LOG_ERROR, "Failed to create tempfile\n");
  83         return c->fd;
  84     }
  85
  86     unlink(buffername);
  87     av_freep(&buffername);
  88
  89     return ffurl_open_whitelist(&c->inner, arg, flags, &h->interrupt_callback,
  90                                 options, h->protocol_whitelist, h->protocol_blacklist, h);
  91 }
  92
  93 static int add_entry(URLContext *h, const unsigned char *buf, int size)
  94 {
  95     Context *c= h->priv_data;
  96     int64_t pos = -1;
  97     int ret;
  98     CacheEntry *entry = NULL, *next[2] = {NULL, NULL};
  99     CacheEntry *entry_ret;
 100     struct AVTreeNode *node = NULL;
 101
 102     //FIXME avoid lseek
 103     pos = lseek(c->fd, 0, SEEK_END);
 104     if (pos < 0) {
 105         ret = AVERROR(errno);
 106         av_log(h, AV_LOG_ERROR, "seek in cache failed\n");
 107         goto fail;
 108     }
 109     c->cache_pos = pos;
 110
 111     ret = write(c->fd, buf, size);
 112     if (ret < 0) {
 113         ret = AVERROR(errno);
 114         av_log(h, AV_LOG_ERROR, "write in cache failed\n");
 115         goto fail;
 116     }
 117     c->cache_pos += ret;
 118
 119     entry = av_tree_find(c->root, &c->logical_pos, cmp, (void**)next);
 120
 121     if (!entry)
 122         entry = next[0];
 123
 124     if (!entry ||
 125         entry->logical_pos  + entry->size != c->logical_pos ||
 126         entry->physical_pos + entry->size != pos
 127     ) {
 128         entry = av_malloc(sizeof(*entry));
 129         node = av_tree_node_alloc();
 130         if (!entry || !node) {
 131             ret = AVERROR(ENOMEM);
 132             goto fail;
 133         }
 134         entry->logical_pos = c->logical_pos;
 135         entry->physical_pos = pos;
 136         entry->size = ret;
 137
 138         entry_ret = av_tree_insert(&c->root, entry, cmp, &node);
 139         if (entry_ret && entry_ret != entry) {
 140             ret = -1;
 141             av_log(h, AV_LOG_ERROR, "av_tree_insert failed\n");
 142             goto fail;
 143         }
 144     } else
 145         entry->size += ret;
 146
 147     return 0;
 148 fail:
 149     //we could truncate the file to pos here if pos >=0 but ftruncate isn't available in VS so
 150     //for simplicty we just leave the file a bit larger
 151     av_free(entry);
 152     av_free(node);
 153     return ret;
 154 }
 155
 156 static int cache_read(URLContext *h, unsigned char *buf, int size)
 157 {
 158     Context *c= h->priv_data;
 159     CacheEntry *entry, *next[2] = {NULL, NULL};
 160     int64_t r;
 161
 162     entry = av_tree_find(c->root, &c->logical_pos, cmp, (void**)next);
 163
 164     if (!entry)
 165         entry = next[0];
 166
 167     if (entry) {
 168         int64_t in_block_pos = c->logical_pos - entry->logical_pos;
 169         av_assert0(entry->logical_pos <= c->logical_pos);
 170         if (in_block_pos < entry->size) {
 171             int64_t physical_target = entry->physical_pos + in_block_pos;
 172
 173             if (c->cache_pos != physical_target) {
 174                 r = lseek(c->fd, physical_target, SEEK_SET);
 175             } else
 176                 r = c->cache_pos;
 177
 178             if (r >= 0) {
 179                 c->cache_pos = r;
 180                 r = read(c->fd, buf, FFMIN(size, entry->size - in_block_pos));
 181             }
 182
 183             if (r > 0) {
 184                 c->cache_pos += r;
 185                 c->logical_pos += r;
 186                 c->cache_hit ++;
 187                 return r;
 188             }
 189         }
 190     }
 191
 192     // Cache miss or some kind of fault with the cache
 193
 194     if (c->logical_pos != c->inner_pos) {
 195         r = ffurl_seek(c->inner, c->logical_pos, SEEK_SET);
 196         if (r<0) {
 197             av_log(h, AV_LOG_ERROR, "Failed to perform internal seek\n");
 198             return r;
 199         }
 200         c->inner_pos = r;
 201     }
 202
 203     r = ffurl_read(c->inner, buf, size);
 204     if (r == 0 && size>0) {
 205         c->is_true_eof = 1;
 206         av_assert0(c->end >= c->logical_pos);
 207     }
 208     if (r<=0)
 209         return r;
 210     c->inner_pos += r;
 211
 212     c->cache_miss ++;
 213
 214     add_entry(h, buf, r);
 215     c->logical_pos += r;
 216     c->end = FFMAX(c->end, c->logical_pos);
 217
 218     return r;
 219 }
 220
 221 static int64_t cache_seek(URLContext *h, int64_t pos, int whence)
 222 {
 223     Context *c= h->priv_data;
 224     int64_t ret;
 225
 226     if (whence == AVSEEK_SIZE) {
 227         pos= ffurl_seek(c->inner, pos, whence);
 228         if(pos <= 0){
 229             pos= ffurl_seek(c->inner, -1, SEEK_END);
 230             if (ffurl_seek(c->inner, c->inner_pos, SEEK_SET) < 0)
 231                 av_log(h, AV_LOG_ERROR, "Inner protocol failed to seekback end : %"PRId64"\n", pos);
 232         }
 233         if (pos > 0)
 234             c->is_true_eof = 1;
 235         c->end = FFMAX(c->end, pos);
 236         return pos;
 237     }
 238
 239     if (whence == SEEK_CUR) {
 240         whence = SEEK_SET;
 241         pos += c->logical_pos;
 242     } else if (whence == SEEK_END && c->is_true_eof) {
 243 resolve_eof:
 244         whence = SEEK_SET;
 245         pos += c->end;
 246     }
 247
 248     if (whence == SEEK_SET && pos >= 0 && pos < c->end) {
 249         //Seems within filesize, assume it will not fail.
 250         c->logical_pos = pos;
 251         return pos;
 252     }
 253
 254     //cache miss
 255     ret= ffurl_seek(c->inner, pos, whence);
 256     if ((whence == SEEK_SET && pos >= c->logical_pos ||
 257          whence == SEEK_END && pos <= 0) && ret < 0) {
 258         if (   (whence == SEEK_SET && c->read_ahead_limit >= pos - c->logical_pos)
 259             || c->read_ahead_limit < 0) {
 260             uint8_t tmp[32768];
 261             while (c->logical_pos < pos || whence == SEEK_END) {
 262                 int size = sizeof(tmp);
 263                 if (whence == SEEK_SET)
 264                     size = FFMIN(sizeof(tmp), pos - c->logical_pos);
 265                 ret = cache_read(h, tmp, size);
 266                 if (ret == 0 && whence == SEEK_END) {
 267                     av_assert0(c->is_true_eof);
 268                     goto resolve_eof;
 269                 }
 270                 if (ret < 0) {
 271                     return ret;
 272                 }
 273             }
 274             return c->logical_pos;
 275         }
 276     }
 277
 278     if (ret >= 0) {
 279         c->logical_pos = ret;
 280         c->end = FFMAX(c->end, ret);
 281     }
 282
 283     return ret;
 284 }
 285
 286 static int enu_free(void *opaque, void *elem)
 287 {
 288     av_free(elem);
 289     return 0;
 290 }
 291
 292 static int cache_close(URLContext *h)
 293 {
 294     Context *c= h->priv_data;
 295
 296     av_log(h, AV_LOG_INFO, "Statistics, cache hits:%"PRId64" cache misses:%"PRId64"\n",
 297            c->cache_hit, c->cache_miss);
 298
 299     close(c->fd);
 300     ffurl_close(c->inner);
 301     av_tree_enumerate(c->root, NULL, NULL, enu_free);
 302     av_tree_destroy(c->root);
 303
 304     return 0;
 305 }
 306
 307 #define OFFSET(x) offsetof(Context, x)
 308 #define D AV_OPT_FLAG_DECODING_PARAM
 309
 310 static const AVOption options[] = {
 311     { "read_ahead_limit", "Amount in bytes that may be read ahead when seeking isn't supported, -1 for unlimited", OFFSET(read_ahead_limit), AV_OPT_TYPE_INT, { .i64 = 65536 }, -1, INT_MAX, D },
 312     {NULL},
 313 };
 314
 315 static const AVClass cache_context_class = {
 316     .class_name = "Cache",
 317     .item_name  = av_default_item_name,
 318     .option     = options,
 319     .version    = LIBAVUTIL_VERSION_INT,
 320 };
 321
 322 const URLProtocol ff_cache_protocol = {
 323     .name                = "cache",
 324     .url_open2           = cache_open,
 325     .url_read            = cache_read,
 326     .url_seek            = cache_seek,
 327     .url_close           = cache_close,
 328     .priv_data_size      = sizeof(Context),
 329     .priv_data_class     = &cache_context_class,
 330 };