libavformat/ipfsgateway.c

   1 /*
   2  * IPFS and IPNS protocol support through IPFS Gateway.
   3  * Copyright (c) 2022 Mark Gaiser
   4  *
   5  * This file is part of FFmpeg.
   6  *
   7  * FFmpeg is free software; you can redistribute it and/or
   8  * modify it under the terms of the GNU Lesser General Public
   9  * License as published by the Free Software Foundation; either
  10  * version 2.1 of the License, or (at your option) any later version.
  11  *
  12  * FFmpeg is distributed in the hope that it will be useful,
  13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  15  * Lesser General Public License for more details.
  16  *
  17  * You should have received a copy of the GNU Lesser General Public
  18  * License along with FFmpeg; if not, write to the Free Software
  19  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
  20  */
  21
  22 #include "libavutil/avstring.h"
  23 #include "libavutil/file_open.h"
  24 #include "libavutil/getenv_utf8.h"
  25 #include "libavutil/mem.h"
  26 #include "libavutil/opt.h"
  27 #include <sys/stat.h>
  28 #include "os_support.h"
  29 #include "url.h"
  30
  31 // Define the posix PATH_MAX if not there already.
  32 // This fixes a compile issue for MSVC.
  33 #ifndef PATH_MAX
  34 #define PATH_MAX 4096
  35 #endif
  36
  37 typedef struct IPFSGatewayContext {
  38     AVClass *class;
  39     URLContext *inner;
  40     // Is filled by the -gateway argument and not changed after.
  41     char *gateway;
  42     // If the above gateway is non null, it will be copied into this buffer.
  43     // Else this buffer will contain the auto detected gateway.
  44     // In either case, the gateway to use will be in this buffer.
  45     char gateway_buffer[PATH_MAX];
  46 } IPFSGatewayContext;
  47
  48 // A best-effort way to find the IPFS gateway.
  49 // Only the most appropriate gateway is set. It's not actually requested
  50 // (http call) to prevent a potential slowdown in startup. A potential timeout
  51 // is handled by the HTTP protocol.
  52 static int populate_ipfs_gateway(URLContext *h)
  53 {
  54     IPFSGatewayContext *c = h->priv_data;
  55     char ipfs_full_data_folder[PATH_MAX];
  56     char ipfs_gateway_file[PATH_MAX];
  57     struct stat st;
  58     int stat_ret = 0;
  59     int ret = AVERROR(EINVAL);
  60     FILE *gateway_file = NULL;
  61     char *env_ipfs_gateway, *env_ipfs_path;
  62
  63     // Test $IPFS_GATEWAY.
  64     env_ipfs_gateway = getenv_utf8("IPFS_GATEWAY");
  65     if (env_ipfs_gateway != NULL) {
  66         int printed = snprintf(c->gateway_buffer, sizeof(c->gateway_buffer),
  67                                "%s", env_ipfs_gateway);
  68         freeenv_utf8(env_ipfs_gateway);
  69         if (printed >= sizeof(c->gateway_buffer)) {
  70             av_log(h, AV_LOG_WARNING,
  71                    "The IPFS_GATEWAY environment variable "
  72                    "exceeds the maximum length. "
  73                    "We allow a max of %zu characters\n",
  74                    sizeof(c->gateway_buffer));
  75             ret = AVERROR(EINVAL);
  76             goto err;
  77         }
  78
  79         ret = 1;
  80         goto err;
  81     } else
  82         av_log(h, AV_LOG_DEBUG, "$IPFS_GATEWAY is empty.\n");
  83
  84     // We need to know the IPFS folder to - eventually - read the contents of
  85     // the "gateway" file which would tell us the gateway to use.
  86     env_ipfs_path = getenv_utf8("IPFS_PATH");
  87     if (env_ipfs_path == NULL) {
  88         int printed;
  89         char *env_home = getenv_utf8("HOME");
  90
  91         av_log(h, AV_LOG_DEBUG, "$IPFS_PATH is empty.\n");
  92
  93         // Try via the home folder.
  94         if (env_home == NULL) {
  95             av_log(h, AV_LOG_WARNING, "$HOME appears to be empty.\n");
  96             ret = AVERROR(EINVAL);
  97             goto err;
  98         }
  99
 100         // Verify the composed path fits.
 101         printed = snprintf(
 102             ipfs_full_data_folder, sizeof(ipfs_full_data_folder),
 103             "%s/.ipfs/", env_home);
 104         freeenv_utf8(env_home);
 105         if (printed >= sizeof(ipfs_full_data_folder)) {
 106             av_log(h, AV_LOG_WARNING,
 107                    "The IPFS data path exceeds the "
 108                    "max path length (%zu)\n",
 109                    sizeof(ipfs_full_data_folder));
 110             ret = AVERROR(EINVAL);
 111             goto err;
 112         }
 113
 114         // Stat the folder.
 115         // It should exist in a default IPFS setup when run as local user.
 116         stat_ret = stat(ipfs_full_data_folder, &st);
 117
 118         if (stat_ret < 0) {
 119             av_log(h, AV_LOG_INFO,
 120                    "Unable to find IPFS folder. We tried:\n"
 121                    "- $IPFS_PATH, which was empty.\n"
 122                    "- $HOME/.ipfs (full uri: %s) which doesn't exist.\n",
 123                    ipfs_full_data_folder);
 124             ret = AVERROR(ENOENT);
 125             goto err;
 126         }
 127     } else {
 128         int printed = snprintf(
 129             ipfs_full_data_folder, sizeof(ipfs_full_data_folder),
 130             "%s", env_ipfs_path);
 131         freeenv_utf8(env_ipfs_path);
 132         if (printed >= sizeof(ipfs_full_data_folder)) {
 133             av_log(h, AV_LOG_WARNING,
 134                    "The IPFS_PATH environment variable "
 135                    "exceeds the maximum length. "
 136                    "We allow a max of %zu characters\n",
 137                    sizeof(c->gateway_buffer));
 138             ret = AVERROR(EINVAL);
 139             goto err;
 140         }
 141     }
 142
 143     // Copy the fully composed gateway path into ipfs_gateway_file.
 144     if (snprintf(ipfs_gateway_file, sizeof(ipfs_gateway_file), "%sgateway",
 145                  ipfs_full_data_folder)
 146         >= sizeof(ipfs_gateway_file)) {
 147         av_log(h, AV_LOG_WARNING,
 148                "The IPFS gateway file path exceeds "
 149                "the max path length (%zu)\n",
 150                sizeof(ipfs_gateway_file));
 151         ret = AVERROR(ENOENT);
 152         goto err;
 153     }
 154
 155     // Get the contents of the gateway file.
 156     gateway_file = avpriv_fopen_utf8(ipfs_gateway_file, "r");
 157     if (!gateway_file) {
 158         av_log(h, AV_LOG_WARNING,
 159                "The IPFS gateway file (full uri: %s) doesn't exist. "
 160                "Is the gateway enabled?\n",
 161                ipfs_gateway_file);
 162         ret = AVERROR(ENOENT);
 163         goto err;
 164     }
 165
 166     // Read a single line (fgets stops at new line mark).
 167     if (!fgets(c->gateway_buffer, sizeof(c->gateway_buffer) - 1, gateway_file)) {
 168         av_log(h, AV_LOG_WARNING, "Unable to read from file (full uri: %s).\n",
 169                ipfs_gateway_file);
 170         ret = AVERROR(ENOENT);
 171         goto err;
 172     }
 173
 174     // Replace first occurrence of end of line with \0
 175     c->gateway_buffer[strcspn(c->gateway_buffer, "\r\n")] = 0;
 176
 177     // If strlen finds anything longer then 0 characters then we have a
 178     // potential gateway url.
 179     if (*c->gateway_buffer == '\0') {
 180         av_log(h, AV_LOG_WARNING,
 181                "The IPFS gateway file (full uri: %s) appears to be empty. "
 182                "Is the gateway started?\n",
 183                ipfs_gateway_file);
 184         ret = AVERROR(EILSEQ);
 185         goto err;
 186     } else {
 187         // We're done, the c->gateway_buffer has something that looks valid.
 188         ret = 1;
 189         goto err;
 190     }
 191
 192 err:
 193     if (gateway_file)
 194         fclose(gateway_file);
 195
 196     return ret;
 197 }
 198
 199 static int translate_ipfs_to_http(URLContext *h, const char *uri, int flags, AVDictionary **options)
 200 {
 201     const char *ipfs_cid;
 202     char *fulluri = NULL;
 203     int ret;
 204     IPFSGatewayContext *c = h->priv_data;
 205
 206     // Test for ipfs://, ipfs:, ipns:// and ipns:. This prefix is stripped from
 207     // the string leaving just the CID in ipfs_cid.
 208     int is_ipfs = av_stristart(uri, "ipfs://", &ipfs_cid);
 209     int is_ipns = av_stristart(uri, "ipns://", &ipfs_cid);
 210
 211     // We must have either ipns or ipfs.
 212     if (!is_ipfs && !is_ipns) {
 213         ret = AVERROR(EINVAL);
 214         av_log(h, AV_LOG_WARNING, "Unsupported url %s\n", uri);
 215         goto err;
 216     }
 217
 218     // If the CID has a length greater then 0 then we assume we have a proper working one.
 219     // It could still be wrong but in that case the gateway should save us and
 220     // ruturn a 403 error. The http protocol handles this.
 221     if (strlen(ipfs_cid) < 1) {
 222         av_log(h, AV_LOG_WARNING, "A CID must be provided.\n");
 223         ret = AVERROR(EILSEQ);
 224         goto err;
 225     }
 226
 227     // Populate c->gateway_buffer with whatever is in c->gateway
 228     if (c->gateway != NULL) {
 229         if (snprintf(c->gateway_buffer, sizeof(c->gateway_buffer), "%s",
 230                      c->gateway)
 231             >= sizeof(c->gateway_buffer)) {
 232             av_log(h, AV_LOG_WARNING,
 233                    "The -gateway parameter is too long. "
 234                    "We allow a max of %zu characters\n",
 235                    sizeof(c->gateway_buffer));
 236             ret = AVERROR(EINVAL);
 237             goto err;
 238         }
 239     } else {
 240         // Populate the IPFS gateway if we have any.
 241         // If not, inform the user how to properly set one.
 242         ret = populate_ipfs_gateway(h);
 243
 244         if (ret < 1) {
 245             av_log(h, AV_LOG_ERROR,
 246                    "IPFS does not appear to be running.\n\n"
 247                    "Installing IPFS locally is recommended to "
 248                    "improve performance and reliability, "
 249                    "and not share all your activity with a single IPFS gateway.\n"
 250                    "There are multiple options to define this gateway.\n"
 251                    "1. Call ffmpeg with a gateway param, "
 252                    "without a trailing slash: -gateway <url>.\n"
 253                    "2. Define an $IPFS_GATEWAY environment variable with the "
 254                    "full HTTP URL to the gateway "
 255                    "without trailing forward slash.\n"
 256                    "3. Define an $IPFS_PATH environment variable "
 257                    "and point it to the IPFS data path "
 258                    "- this is typically ~/.ipfs\n");
 259             ret = AVERROR(EINVAL);
 260             goto err;
 261         }
 262     }
 263
 264     // Test if the gateway starts with either http:// or https://
 265     if (av_stristart(c->gateway_buffer, "http://", NULL) == 0
 266         && av_stristart(c->gateway_buffer, "https://", NULL) == 0) {
 267         av_log(h, AV_LOG_WARNING,
 268                "The gateway URL didn't start with http:// or "
 269                "https:// and is therefore invalid.\n");
 270         ret = AVERROR(EILSEQ);
 271         goto err;
 272     }
 273
 274     // Concatenate the url.
 275     // This ends up with something like: http://localhost:8080/ipfs/Qm.....
 276     // The format of "%s%s%s%s" is the following:
 277     // 1st %s = The gateway.
 278     // 2nd %s = If the gateway didn't end in a slash, add a "/". Otherwise it's an empty string
 279     // 3rd %s = Either ipns/ or ipfs/.
 280     // 4th %s = The IPFS CID (Qm..., bafy..., ...).
 281     fulluri = av_asprintf("%s%s%s%s",
 282                           c->gateway_buffer,
 283                           (c->gateway_buffer[strlen(c->gateway_buffer) - 1] == '/') ? "" : "/",
 284                           (is_ipns) ? "ipns/" : "ipfs/",
 285                           ipfs_cid);
 286
 287     if (!fulluri) {
 288         av_log(h, AV_LOG_ERROR, "Failed to compose the URL\n");
 289         ret = AVERROR(ENOMEM);
 290         goto err;
 291     }
 292
 293     // Pass the URL back to FFmpeg's protocol handler.
 294     ret = ffurl_open_whitelist(&c->inner, fulluri, flags,
 295                                &h->interrupt_callback, options,
 296                                h->protocol_whitelist,
 297                                h->protocol_blacklist, h);
 298     if (ret < 0) {
 299         av_log(h, AV_LOG_WARNING, "Unable to open resource: %s\n", fulluri);
 300         goto err;
 301     }
 302
 303 err:
 304     av_free(fulluri);
 305     return ret;
 306 }
 307
 308 static int ipfs_read(URLContext *h, unsigned char *buf, int size)
 309 {
 310     IPFSGatewayContext *c = h->priv_data;
 311     return ffurl_read(c->inner, buf, size);
 312 }
 313
 314 static int64_t ipfs_seek(URLContext *h, int64_t pos, int whence)
 315 {
 316     IPFSGatewayContext *c = h->priv_data;
 317     return ffurl_seek(c->inner, pos, whence);
 318 }
 319
 320 static int ipfs_close(URLContext *h)
 321 {
 322     IPFSGatewayContext *c = h->priv_data;
 323     return ffurl_closep(&c->inner);
 324 }
 325
 326 #define OFFSET(x) offsetof(IPFSGatewayContext, x)
 327
 328 static const AVOption options[] = {
 329     {"gateway", "The gateway to ask for IPFS data.", OFFSET(gateway), AV_OPT_TYPE_STRING, {.str = NULL}, 0, 0, AV_OPT_FLAG_DECODING_PARAM},
 330     {NULL},
 331 };
 332
 333 static const AVClass ipfs_gateway_context_class = {
 334     .class_name     = "IPFS Gateway",
 335     .item_name      = av_default_item_name,
 336     .option         = options,
 337     .version        = LIBAVUTIL_VERSION_INT,
 338 };
 339
 340 const URLProtocol ff_ipfs_gateway_protocol = {
 341     .name               = "ipfs",
 342     .url_open2          = translate_ipfs_to_http,
 343     .url_read           = ipfs_read,
 344     .url_seek           = ipfs_seek,
 345     .url_close          = ipfs_close,
 346     .priv_data_size     = sizeof(IPFSGatewayContext),
 347     .priv_data_class    = &ipfs_gateway_context_class,
 348 };
 349
 350 const URLProtocol ff_ipns_gateway_protocol = {
 351     .name               = "ipns",
 352     .url_open2          = translate_ipfs_to_http,
 353     .url_read           = ipfs_read,
 354     .url_seek           = ipfs_seek,
 355     .url_close          = ipfs_close,
 356     .priv_data_size     = sizeof(IPFSGatewayContext),
 357     .priv_data_class    = &ipfs_gateway_context_class,
 358 };