nuttx-apps/system/nxpkg/pkg_repo.c

/****************************************************************************
 * apps/system/nxpkg/pkg_repo.c
 *
 * SPDX-License-Identifier: Apache-2.0
 *
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.  The
 * ASF licenses this file to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance with the
 * License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
 * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
 * License for the specific language governing permissions and limitations
 * under the License.
 *
 ****************************************************************************/

/****************************************************************************
 * Included Files
 ****************************************************************************/

#include <ctype.h>
#include <errno.h>
#include <fcntl.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <strings.h>
#include <unistd.h>
#include <sys/stat.h>

#include <nuttx/config.h>

#include <netutils/cJSON.h>

#ifdef CONFIG_NETUTILS_WEBCLIENT
#  include "netutils/webclient.h"
#endif

#include "pkg.h"

/****************************************************************************
 * Pre-processor Definitions
 ****************************************************************************/

/* Balance network throughput against the 16 KiB caller stacks. */

#define PKG_REPO_FETCH_BUFFER_SIZE 4096
#define PKG_REPO_HTTP              "http://"
#define PKG_REPO_HTTPS             "https://"
#define PKG_REPO_SOURCE_KEY        "_nxpkg_source"

/****************************************************************************
 * Private Types
 ****************************************************************************/

#ifdef CONFIG_NETUTILS_WEBCLIENT
struct pkg_fetch_context_s
{
  int fd;
  size_t total;
};
#endif

/****************************************************************************
 * Private Functions
 ****************************************************************************/

static int pkg_repo_copy_string(FAR char *buffer, size_t size,
                                FAR const char *value)
{
  int ret;

  ret = snprintf(buffer, size, "%s", value);
  if (ret < 0)
    {
      return ret;
    }

  return (size_t)ret >= size ? -ENAMETOOLONG : 0;
}

static int pkg_repo_source_base(FAR char *buffer, size_t size,
                                FAR const char *source)
{
  FAR const char *slash;
  size_t length;

  slash = strrchr(source, '/');
  if (slash == NULL)
    {
      return pkg_repo_copy_string(buffer, size, ".");
    }

  length = (size_t)(slash - source);
  if (length == 0)
    {
      length = 1;
    }

  if (length >= size)
    {
      return -ENAMETOOLONG;
    }

  memcpy(buffer, source, length);
  buffer[length] = '\0';
  return 0;
}

/****************************************************************************
 * Name: pkg_validate_artifact_relative
 *
 * Description:
 *   Reject absolute paths and parent-directory traversal.
 *
 ****************************************************************************/

static bool pkg_validate_artifact_relative(FAR const char *value)
{
  FAR const char *p;

  if (value == NULL || value[0] == '\0' || value[0] == '/')
    {
      return false;
    }

  p = value;
  while ((p = strstr(p, "..")) != NULL)
    {
      bool at_start = p == value || *(p - 1) == '/';
      bool at_end = p[2] == '\0' || p[2] == '/';

      if (at_start && at_end)
        {
          return false;
        }

      p++;
    }

  return true;
}

static int pkg_repo_read_source(FAR char *buffer, size_t size)
{
  FAR cJSON *root;
  FAR cJSON *source;
  FAR char *text = NULL;
  char path[PATH_MAX];
  size_t length;
  int ret;

  ret = pkg_store_format_index_path(path, sizeof(path));
  if (ret < 0)
    {
      return ret;
    }

  ret = pkg_store_read_text(path, &text);
  if (ret < 0)
    {
      return ret;
    }

  root = cJSON_Parse(text);
  pkg_free(text);
  if (root == NULL)
    {
      return -EINVAL;
    }

  source = cJSON_GetObjectItemCaseSensitive(root, PKG_REPO_SOURCE_KEY);
  if (!cJSON_IsString(source) || source->valuestring == NULL)
    {
      cJSON_Delete(root);

      /* Fall back to the legacy source sidecar. */

      ret = pkg_store_format_repo_source_path(path, sizeof(path));
      if (ret < 0)
        {
          return ret;
        }

      ret = pkg_store_read_text(path, &text);
      if (ret < 0)
        {
          return ret;
        }

      length = strlen(text);
      while (length > 0 && isspace((unsigned char)text[length - 1]))
        {
          text[--length] = '\0';
        }

      ret = pkg_repo_copy_string(buffer, size, text);
      pkg_free(text);
      return ret;
    }

  ret = pkg_repo_copy_string(buffer, size, source->valuestring);
  cJSON_Delete(root);
  return ret;
}

static int pkg_repo_attach_source(FAR char **text,
                                  FAR const char *source_value)
{
  FAR cJSON *root;
  FAR cJSON *wrapper;
  FAR char *updated;

  root = cJSON_Parse(*text);
  if (root == NULL)
    {
      return -EINVAL;
    }

  if (cJSON_IsArray(root))
    {
      wrapper = cJSON_CreateObject();
      if (wrapper == NULL)
        {
          cJSON_Delete(root);
          return -ENOMEM;
        }

      cJSON_AddItemToObject(wrapper, "packages", root);
      if (cJSON_GetObjectItemCaseSensitive(wrapper, "packages") != root)
        {
          cJSON_Delete(root);
          cJSON_Delete(wrapper);
          return -ENOMEM;
        }

      root = wrapper;
    }
  else if (!cJSON_IsObject(root))
    {
      cJSON_Delete(root);
      return -EINVAL;
    }

  while (cJSON_GetObjectItemCaseSensitive(root,
                                          PKG_REPO_SOURCE_KEY) != NULL)
    {
      cJSON_DeleteItemFromObjectCaseSensitive(root, PKG_REPO_SOURCE_KEY);
    }

  if (cJSON_AddStringToObject(root, PKG_REPO_SOURCE_KEY,
                             source_value) == NULL)
    {
      cJSON_Delete(root);
      return -ENOMEM;
    }

  updated = cJSON_PrintUnformatted(root);
  cJSON_Delete(root);
  if (updated == NULL)
    {
      return -ENOMEM;
    }

  pkg_free(*text);
  *text = updated;
  return 0;
}

#ifdef CONFIG_NETUTILS_WEBCLIENT
static int pkg_repo_sink(FAR char **buffer, int offset, int datend,
                         FAR int *buflen, FAR void *arg)
{
  FAR struct pkg_fetch_context_s *ctx;
  size_t remaining;
  FAR char *cursor;

  UNUSED(buffer);
  UNUSED(buflen);

  ctx = arg;
  cursor = &(*buffer)[offset];
  remaining = (size_t)(datend - offset);

  /* Enforce the download limit before writing this chunk. */

  if (remaining > 0 &&
      (ctx->total > PKG_DOWNLOAD_MAX_SIZE ||
       remaining > PKG_DOWNLOAD_MAX_SIZE - ctx->total))
    {
      return -EFBIG;
    }

  ctx->total += remaining;

  while (remaining > 0)
    {
      ssize_t nwritten;

      nwritten = write(ctx->fd, cursor, remaining);
      if (nwritten < 0)
        {
          if (errno == EINTR)
            {
              continue;
            }

          return -errno;
        }

      if (nwritten == 0)
        {
          return -EIO;
        }

      cursor += nwritten;
      remaining -= (size_t)nwritten;
    }

  return 0;
}

static int pkg_repo_fetch_url(FAR const char *url, FAR const char *dest)
{
  struct pkg_fetch_context_s fetch;
  struct webclient_context client;
  char reason[64];
  char buffer[PKG_REPO_FETCH_BUFFER_SIZE];
  int ret;

  fetch.fd = open(dest, O_WRONLY | O_CREAT | O_TRUNC, 0644);
  if (fetch.fd < 0)
    {
      return -errno;
    }

  fetch.total = 0;
  webclient_set_defaults(&client);
  client.method = "GET";
  client.url = url;
  client.buffer = buffer;
  client.buflen = sizeof(buffer);
  client.sink_callback = pkg_repo_sink;
  client.sink_callback_arg = &fetch;
  client.http_reason = reason;
  client.http_reason_len = sizeof(reason);

  ret = webclient_perform(&client);
  if (ret < 0)
    {
      close(fetch.fd);
      unlink(dest);
      return ret;
    }

  if (client.http_status / 100 != 2)
    {
      close(fetch.fd);
      unlink(dest);
      return -EPROTO;
    }

  ret = close(fetch.fd);
  if (ret < 0)
    {
      ret = -errno;
      unlink(dest);
      return ret;
    }

  return 0;
}
#endif

static int pkg_resolve_relative_source(FAR char *buffer, size_t size,
                                       FAR const char *relative)
{
  char source[PATH_MAX];
  char base[PATH_MAX];
  int ret;

  if (buffer == NULL || relative == NULL || relative[0] == '\0')
    {
      return -EINVAL;
    }

  if (pkg_source_is_url(relative))
    {
      return pkg_repo_copy_string(buffer, size, relative);
    }

  if (!pkg_validate_artifact_relative(relative))
    {
      return -EINVAL;
    }

  ret = pkg_repo_read_source(source, sizeof(source));
  if (ret >= 0)
    {
      ret = pkg_repo_source_base(base, sizeof(base), source);
      if (ret < 0)
        {
          return ret;
        }

      ret = snprintf(buffer, size, "%s/%s", base, relative);
      if (ret < 0)
        {
          return ret;
        }

      return (size_t)ret >= size ? -ENAMETOOLONG : 0;
    }

  ret = snprintf(buffer, size, "%s/%s", PKG_REPO_DIR, relative);
  if (ret < 0)
    {
      return ret;
    }

  return (size_t)ret >= size ? -ENAMETOOLONG : 0;
}

/****************************************************************************
 * Public Functions
 ****************************************************************************/

bool pkg_source_is_url(FAR const char *source)
{
  if (source == NULL)
    {
      return false;
    }

  return strncasecmp(source, PKG_REPO_HTTP, strlen(PKG_REPO_HTTP)) == 0 ||
         strncasecmp(source, PKG_REPO_HTTPS, strlen(PKG_REPO_HTTPS)) == 0;
}

int pkg_resolve_artifact_source(FAR char *buffer, size_t size,
                                FAR const struct pkg_manifest_s *manifest)
{
  if (manifest == NULL)
    {
      return -EINVAL;
    }

  return pkg_resolve_relative_source(buffer, size, manifest->artifact);
}

int pkg_resolve_icon_source(FAR char *buffer, size_t size,
                            FAR const struct pkg_manifest_s *manifest)
{
  if (manifest == NULL)
    {
      return -EINVAL;
    }

  return pkg_resolve_relative_source(buffer, size, manifest->icon);
}

int pkg_acquire_source(FAR const char *source, FAR const char *dest)
{
  if (source == NULL || dest == NULL)
    {
      return -EINVAL;
    }

  if (pkg_source_is_url(source))
    {
#ifdef CONFIG_NETUTILS_WEBCLIENT
      return pkg_repo_fetch_url(source, dest);
#else
      return -ENOSYS;
#endif
    }

  return pkg_store_copy_file(source, dest);
}

/****************************************************************************
 * Name: pkg_repo_acquire_sync_lock
 *
 * Description:
 *   Serialize catalog synchronization so concurrent downloads cannot
 *   commit out of order.
 *
 ****************************************************************************/

static int pkg_repo_acquire_sync_lock(FAR char *path, size_t size)
{
  int ret;
  int tries;

  ret = snprintf(path, size, PKG_ROOT_DIR "/sync.lk");
  if (ret < 0)
    {
      return ret;
    }

  if ((size_t)ret >= size)
    {
      return -ENAMETOOLONG;
    }

  for (tries = 0; tries < 100; tries++)
    {
      ret = pkg_lock_create(path);
      if (ret == 0)
        {
          return 0;
        }

      if (ret != -EEXIST)
        {
          return ret;
        }

      pkg_reclaim_stale_lock(path);
      usleep(20 * 1000);
    }

  return -EBUSY;
}

int pkg_sync(FAR const char *source)
{
  FAR struct pkg_index_s *index = NULL;
  FAR char *text = NULL;
  FAR char *tmp = NULL;
  FAR char *index_path = NULL;
  FAR char *lock;
  bool remove_tmp = false;
  int ret;

  if (source == NULL || source[0] == '\0')
    {
      pkg_error("sync requires a non-empty index source");
      return -EINVAL;
    }

  ret = pkg_store_prepare_layout();
  if (ret < 0)
    {
      pkg_error("unable to prepare package layout: %d", ret);
      return ret;
    }

  lock = pkg_malloc(PATH_MAX);
  if (lock == NULL)
    {
      pkg_error("unable to allocate sync lock path buffer");
      return -ENOMEM;
    }

  ret = pkg_repo_acquire_sync_lock(lock, PATH_MAX);
  if (ret < 0)
    {
      pkg_error("unable to acquire sync lock: %d", ret);
      pkg_free(lock);
      return ret;
    }

  index = pkg_zalloc(sizeof(*index));
  tmp = pkg_malloc(PATH_MAX);
  index_path = pkg_malloc(PATH_MAX);
  if (index == NULL || tmp == NULL || index_path == NULL)
    {
      pkg_error("unable to allocate index metadata buffer");
      ret = -ENOMEM;
      goto out;
    }

  /* Use a per-process FAT-compatible staging name. */

  ret = snprintf(tmp, PATH_MAX, "%s/s%u.jsn", PKG_TMP_DIR,
                 (unsigned int)getpid());
  if (ret < 0 || (size_t)ret >= PATH_MAX)
    {
      pkg_error("temporary sync path is too long");
      ret = -ENAMETOOLONG;
      goto out;
    }

  ret = pkg_acquire_source(source, tmp);
  if (ret < 0)
    {
      pkg_error("unable to fetch index source '%s': %d", source, ret);
      goto out;
    }

  remove_tmp = true;
  ret = pkg_metadata_load_index_path(tmp, index);
  if (ret < 0)
    {
      pkg_error("downloaded index is invalid: %d", ret);
      goto out;
    }

  ret = pkg_store_read_text(tmp, &text);
  if (ret < 0)
    {
      pkg_error("unable to read fetched index: %d", ret);
      goto out;
    }

  /* Commit the catalog and its source in one atomic file replacement. */

  ret = pkg_repo_attach_source(&text, source);
  if (ret < 0)
    {
      pkg_error("unable to record repository source: %d", ret);
      goto out;
    }

  ret = pkg_store_format_index_path(index_path, PATH_MAX);
  if (ret < 0)
    {
      pkg_error("unable to resolve local index path: %d", ret);
      goto out;
    }

  ret = pkg_store_write_text_atomic(index_path, text);
  if (ret < 0)
    {
      pkg_error("unable to write local index: %d", ret);
      goto out;
    }

  pkg_info("synced package index from %s", source);
  ret = 0;

out:
  if (remove_tmp)
    {
      pkg_store_remove_file(tmp);
    }

  pkg_free(text);
  pkg_free(index);
  pkg_free(tmp);
  pkg_free(index_path);
  pkg_lock_remove(lock);
  pkg_free(lock);
  return ret;
}

int pkg_available(FAR FILE *stream)
{
  FAR struct pkg_index_s *index;
  FAR const char *arch;
  FAR const char *compat;
  size_t i;
  int ret;

  if (stream == NULL)
    {
      return EXIT_FAILURE;
    }

  index = pkg_zalloc(sizeof(*index));
  if (index == NULL)
    {
      pkg_error("unable to allocate index metadata buffer");
      return EXIT_FAILURE;
    }

  ret = pkg_store_prepare_layout();
  if (ret < 0)
    {
      pkg_free(index);
      pkg_error("unable to prepare package layout: %d", ret);
      return EXIT_FAILURE;
    }

  ret = pkg_metadata_load_index(index);
  if (ret < 0)
    {
      pkg_free(index);
      pkg_error("unable to load package index: %d", ret);
      return EXIT_FAILURE;
    }

  arch = pkg_runtime_arch();
  compat = pkg_runtime_compat();

  for (i = 0; i < index->count; i++)
    {
      FAR const struct pkg_manifest_s *manifest = &index->manifests[i];
      FAR const struct pkg_manifest_s *latest;

      if (strcmp(manifest->arch, arch) != 0 ||
          strcmp(manifest->compat, compat) != 0)
        {
          continue;
        }

      latest = pkg_metadata_find_latest(index, manifest->name);
      if (latest != manifest)
        {
          continue;
        }

      fprintf(stream,
              "%s version=%s type=%s arch=%s compat=%s artifact=%s\n",
              manifest->name,
              manifest->version,
              pkg_manifest_type_str(manifest->type),
              manifest->arch,
              manifest->compat,
              manifest->artifact);
    }

  pkg_free(index);
  return EXIT_SUCCESS;
}