system/nxpkg: fetch repositories over HTTP

Download catalogs and artifacts with bounded buffers and atomic staging. Prepare storage before taking the lock so first-run syncs are safe.

Signed-off-by: aviralgarg05 <gargaviral99@gmail.com>

system/nxpkg/CMakeLists.txt

@@ -38,6 +38,7 @@ if(CONFIG_SYSTEM_NXPKG)
     pkg_log.c
     pkg_manifest.c
     pkg_metadata.c
+    pkg_repo.c
     pkg_store.c
     pkg_txn.c)
 endif()

system/nxpkg/Kconfig

@@ -29,4 +29,15 @@ config SYSTEM_NXPKG_STACKSIZE
 	int "'nxpkg' stack size"
 	default 16384
 
+config SYSTEM_NXPKG_ROOT
+	string "'nxpkg' storage root"
+	default "/var/lib/nxpkg"
+	---help---
+		Base directory used by nxpkg for its local index, installed
+		metadata, temporary downloads, and package payload storage.
+		The default follows the conventional persistent application-data
+		location.  A board must mount persistent storage at /var or set
+		this to another persistent location, such as /mnt/sdcard/nxpkg,
+		if packages need to survive a reset.
+
 endif

system/nxpkg/Makefile

@@ -28,7 +28,7 @@ STACKSIZE = $(CONFIG_SYSTEM_NXPKG_STACKSIZE)
 MODULE    = $(CONFIG_SYSTEM_NXPKG)
 
 CSRCS = pkg_compat.c pkg_hash.c pkg_install.c pkg_log.c pkg_manifest.c
-CSRCS += pkg_metadata.c pkg_store.c pkg_txn.c
+CSRCS += pkg_metadata.c pkg_repo.c pkg_store.c pkg_txn.c
 MAINSRC = pkg_main.c
 
 include $(APPDIR)/Application.mk

system/nxpkg/pkg_repo.c

@@ -0,0 +1,721 @@
+/****************************************************************************
+ * apps/system/nxpkg/pkg_repo.c
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.  The
+ * ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the
+ * License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ ****************************************************************************/
+
+/****************************************************************************
+ * Included Files
+ ****************************************************************************/
+
+#include <ctype.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <strings.h>
+#include <unistd.h>
+#include <sys/stat.h>
+
+#include <nuttx/config.h>
+
+#include <netutils/cJSON.h>
+
+#ifdef CONFIG_NETUTILS_WEBCLIENT
+#  include "netutils/webclient.h"
+#endif
+
+#include "pkg.h"
+
+/****************************************************************************
+ * Pre-processor Definitions
+ ****************************************************************************/
+
+/* Balance network throughput against the 16 KiB caller stacks. */
+
+#define PKG_REPO_FETCH_BUFFER_SIZE 4096
+#define PKG_REPO_HTTP              "http://"
+#define PKG_REPO_HTTPS             "https://"
+#define PKG_REPO_SOURCE_KEY        "_nxpkg_source"
+
+/****************************************************************************
+ * Private Types
+ ****************************************************************************/
+
+#ifdef CONFIG_NETUTILS_WEBCLIENT
+struct pkg_fetch_context_s
+{
+  int fd;
+  size_t total;
+};
+#endif
+
+/****************************************************************************
+ * Private Functions
+ ****************************************************************************/
+
+static int pkg_repo_copy_string(FAR char *buffer, size_t size,
+                                FAR const char *value)
+{
+  int ret;
+
+  ret = snprintf(buffer, size, "%s", value);
+  if (ret < 0)
+    {
+      return ret;
+    }
+
+  return (size_t)ret >= size ? -ENAMETOOLONG : 0;
+}
+
+static int pkg_repo_source_base(FAR char *buffer, size_t size,
+                                FAR const char *source)
+{
+  FAR const char *slash;
+  size_t length;
+
+  slash = strrchr(source, '/');
+  if (slash == NULL)
+    {
+      return pkg_repo_copy_string(buffer, size, ".");
+    }
+
+  length = (size_t)(slash - source);
+  if (length == 0)
+    {
+      length = 1;
+    }
+
+  if (length >= size)
+    {
+      return -ENAMETOOLONG;
+    }
+
+  memcpy(buffer, source, length);
+  buffer[length] = '\0';
+  return 0;
+}
+
+/****************************************************************************
+ * Name: pkg_validate_artifact_relative
+ *
+ * Description:
+ *   Reject absolute paths and parent-directory traversal.
+ *
+ ****************************************************************************/
+
+static bool pkg_validate_artifact_relative(FAR const char *value)
+{
+  FAR const char *p;
+
+  if (value == NULL || value[0] == '\0' || value[0] == '/')
+    {
+      return false;
+    }
+
+  p = value;
+  while ((p = strstr(p, "..")) != NULL)
+    {
+      bool at_start = p == value || *(p - 1) == '/';
+      bool at_end = p[2] == '\0' || p[2] == '/';
+
+      if (at_start && at_end)
+        {
+          return false;
+        }
+
+      p++;
+    }
+
+  return true;
+}
+
+static int pkg_repo_read_source(FAR char *buffer, size_t size)
+{
+  FAR cJSON *root;
+  FAR cJSON *source;
+  FAR char *text = NULL;
+  char path[PATH_MAX];
+  size_t length;
+  int ret;
+
+  ret = pkg_store_format_index_path(path, sizeof(path));
+  if (ret < 0)
+    {
+      return ret;
+    }
+
+  ret = pkg_store_read_text(path, &text);
+  if (ret < 0)
+    {
+      return ret;
+    }
+
+  root = cJSON_Parse(text);
+  pkg_free(text);
+  if (root == NULL)
+    {
+      return -EINVAL;
+    }
+
+  source = cJSON_GetObjectItemCaseSensitive(root, PKG_REPO_SOURCE_KEY);
+  if (!cJSON_IsString(source) || source->valuestring == NULL)
+    {
+      cJSON_Delete(root);
+
+      /* Fall back to the legacy source sidecar. */
+
+      ret = pkg_store_format_repo_source_path(path, sizeof(path));
+      if (ret < 0)
+        {
+          return ret;
+        }
+
+      ret = pkg_store_read_text(path, &text);
+      if (ret < 0)
+        {
+          return ret;
+        }
+
+      length = strlen(text);
+      while (length > 0 && isspace((unsigned char)text[length - 1]))
+        {
+          text[--length] = '\0';
+        }
+
+      ret = pkg_repo_copy_string(buffer, size, text);
+      pkg_free(text);
+      return ret;
+    }
+
+  ret = pkg_repo_copy_string(buffer, size, source->valuestring);
+  cJSON_Delete(root);
+  return ret;
+}
+
+static int pkg_repo_attach_source(FAR char **text,
+                                  FAR const char *source_value)
+{
+  FAR cJSON *root;
+  FAR cJSON *wrapper;
+  FAR char *updated;
+
+  root = cJSON_Parse(*text);
+  if (root == NULL)
+    {
+      return -EINVAL;
+    }
+
+  if (cJSON_IsArray(root))
+    {
+      wrapper = cJSON_CreateObject();
+      if (wrapper == NULL)
+        {
+          cJSON_Delete(root);
+          return -ENOMEM;
+        }
+
+      cJSON_AddItemToObject(wrapper, "packages", root);
+      if (cJSON_GetObjectItemCaseSensitive(wrapper, "packages") != root)
+        {
+          cJSON_Delete(root);
+          cJSON_Delete(wrapper);
+          return -ENOMEM;
+        }
+
+      root = wrapper;
+    }
+  else if (!cJSON_IsObject(root))
+    {
+      cJSON_Delete(root);
+      return -EINVAL;
+    }
+
+  while (cJSON_GetObjectItemCaseSensitive(root,
+                                          PKG_REPO_SOURCE_KEY) != NULL)
+    {
+      cJSON_DeleteItemFromObjectCaseSensitive(root, PKG_REPO_SOURCE_KEY);
+    }
+
+  if (cJSON_AddStringToObject(root, PKG_REPO_SOURCE_KEY,
+                             source_value) == NULL)
+    {
+      cJSON_Delete(root);
+      return -ENOMEM;
+    }
+
+  updated = cJSON_PrintUnformatted(root);
+  cJSON_Delete(root);
+  if (updated == NULL)
+    {
+      return -ENOMEM;
+    }
+
+  pkg_free(*text);
+  *text = updated;
+  return 0;
+}
+
+#ifdef CONFIG_NETUTILS_WEBCLIENT
+static int pkg_repo_sink(FAR char **buffer, int offset, int datend,
+                         FAR int *buflen, FAR void *arg)
+{
+  FAR struct pkg_fetch_context_s *ctx;
+  size_t remaining;
+  FAR char *cursor;
+
+  UNUSED(buffer);
+  UNUSED(buflen);
+
+  ctx = arg;
+  cursor = &(*buffer)[offset];
+  remaining = (size_t)(datend - offset);
+
+  /* Enforce the download limit before writing this chunk. */
+
+  if (remaining > 0 &&
+      (ctx->total > PKG_DOWNLOAD_MAX_SIZE ||
+       remaining > PKG_DOWNLOAD_MAX_SIZE - ctx->total))
+    {
+      return -EFBIG;
+    }
+
+  ctx->total += remaining;
+
+  while (remaining > 0)
+    {
+      ssize_t nwritten;
+
+      nwritten = write(ctx->fd, cursor, remaining);
+      if (nwritten < 0)
+        {
+          if (errno == EINTR)
+            {
+              continue;
+            }
+
+          return -errno;
+        }
+
+      if (nwritten == 0)
+        {
+          return -EIO;
+        }
+
+      cursor += nwritten;
+      remaining -= (size_t)nwritten;
+    }
+
+  return 0;
+}
+
+static int pkg_repo_fetch_url(FAR const char *url, FAR const char *dest)
+{
+  struct pkg_fetch_context_s fetch;
+  struct webclient_context client;
+  char reason[64];
+  char buffer[PKG_REPO_FETCH_BUFFER_SIZE];
+  int ret;
+
+  fetch.fd = open(dest, O_WRONLY | O_CREAT | O_TRUNC, 0644);
+  if (fetch.fd < 0)
+    {
+      return -errno;
+    }
+
+  fetch.total = 0;
+  webclient_set_defaults(&client);
+  client.method = "GET";
+  client.url = url;
+  client.buffer = buffer;
+  client.buflen = sizeof(buffer);
+  client.sink_callback = pkg_repo_sink;
+  client.sink_callback_arg = &fetch;
+  client.http_reason = reason;
+  client.http_reason_len = sizeof(reason);
+
+  ret = webclient_perform(&client);
+  if (ret < 0)
+    {
+      close(fetch.fd);
+      unlink(dest);
+      return ret;
+    }
+
+  if (client.http_status / 100 != 2)
+    {
+      close(fetch.fd);
+      unlink(dest);
+      return -EPROTO;
+    }
+
+  ret = close(fetch.fd);
+  if (ret < 0)
+    {
+      ret = -errno;
+      unlink(dest);
+      return ret;
+    }
+
+  return 0;
+}
+#endif
+
+static int pkg_resolve_relative_source(FAR char *buffer, size_t size,
+                                       FAR const char *relative)
+{
+  char source[PATH_MAX];
+  char base[PATH_MAX];
+  int ret;
+
+  if (buffer == NULL || relative == NULL || relative[0] == '\0')
+    {
+      return -EINVAL;
+    }
+
+  if (pkg_source_is_url(relative))
+    {
+      return pkg_repo_copy_string(buffer, size, relative);
+    }
+
+  if (!pkg_validate_artifact_relative(relative))
+    {
+      return -EINVAL;
+    }
+
+  ret = pkg_repo_read_source(source, sizeof(source));
+  if (ret >= 0)
+    {
+      ret = pkg_repo_source_base(base, sizeof(base), source);
+      if (ret < 0)
+        {
+          return ret;
+        }
+
+      ret = snprintf(buffer, size, "%s/%s", base, relative);
+      if (ret < 0)
+        {
+          return ret;
+        }
+
+      return (size_t)ret >= size ? -ENAMETOOLONG : 0;
+    }
+
+  ret = snprintf(buffer, size, "%s/%s", PKG_REPO_DIR, relative);
+  if (ret < 0)
+    {
+      return ret;
+    }
+
+  return (size_t)ret >= size ? -ENAMETOOLONG : 0;
+}
+
+/****************************************************************************
+ * Public Functions
+ ****************************************************************************/
+
+bool pkg_source_is_url(FAR const char *source)
+{
+  if (source == NULL)
+    {
+      return false;
+    }
+
+  return strncasecmp(source, PKG_REPO_HTTP, strlen(PKG_REPO_HTTP)) == 0 ||
+         strncasecmp(source, PKG_REPO_HTTPS, strlen(PKG_REPO_HTTPS)) == 0;
+}
+
+int pkg_resolve_artifact_source(FAR char *buffer, size_t size,
+                                FAR const struct pkg_manifest_s *manifest)
+{
+  if (manifest == NULL)
+    {
+      return -EINVAL;
+    }
+
+  return pkg_resolve_relative_source(buffer, size, manifest->artifact);
+}
+
+int pkg_resolve_icon_source(FAR char *buffer, size_t size,
+                            FAR const struct pkg_manifest_s *manifest)
+{
+  if (manifest == NULL)
+    {
+      return -EINVAL;
+    }
+
+  return pkg_resolve_relative_source(buffer, size, manifest->icon);
+}
+
+int pkg_acquire_source(FAR const char *source, FAR const char *dest)
+{
+  if (source == NULL || dest == NULL)
+    {
+      return -EINVAL;
+    }
+
+  if (pkg_source_is_url(source))
+    {
+#ifdef CONFIG_NETUTILS_WEBCLIENT
+      return pkg_repo_fetch_url(source, dest);
+#else
+      return -ENOSYS;
+#endif
+    }
+
+  return pkg_store_copy_file(source, dest);
+}
+
+/****************************************************************************
+ * Name: pkg_repo_acquire_sync_lock
+ *
+ * Description:
+ *   Serialize catalog synchronization so concurrent downloads cannot
+ *   commit out of order.
+ *
+ ****************************************************************************/
+
+static int pkg_repo_acquire_sync_lock(FAR char *path, size_t size)
+{
+  int ret;
+  int tries;
+
+  ret = snprintf(path, size, PKG_ROOT_DIR "/sync.lk");
+  if (ret < 0)
+    {
+      return ret;
+    }
+
+  if ((size_t)ret >= size)
+    {
+      return -ENAMETOOLONG;
+    }
+
+  for (tries = 0; tries < 100; tries++)
+    {
+      ret = pkg_lock_create(path);
+      if (ret == 0)
+        {
+          return 0;
+        }
+
+      if (ret != -EEXIST)
+        {
+          return ret;
+        }
+
+      pkg_reclaim_stale_lock(path);
+      usleep(20 * 1000);
+    }
+
+  return -EBUSY;
+}
+
+int pkg_sync(FAR const char *source)
+{
+  FAR struct pkg_index_s *index = NULL;
+  FAR char *text = NULL;
+  FAR char *tmp = NULL;
+  FAR char *index_path = NULL;
+  FAR char *lock;
+  bool remove_tmp = false;
+  int ret;
+
+  if (source == NULL || source[0] == '\0')
+    {
+      pkg_error("sync requires a non-empty index source");
+      return -EINVAL;
+    }
+
+  ret = pkg_store_prepare_layout();
+  if (ret < 0)
+    {
+      pkg_error("unable to prepare package layout: %d", ret);
+      return ret;
+    }
+
+  lock = pkg_malloc(PATH_MAX);
+  if (lock == NULL)
+    {
+      pkg_error("unable to allocate sync lock path buffer");
+      return -ENOMEM;
+    }
+
+  ret = pkg_repo_acquire_sync_lock(lock, PATH_MAX);
+  if (ret < 0)
+    {
+      pkg_error("unable to acquire sync lock: %d", ret);
+      pkg_free(lock);
+      return ret;
+    }
+
+  index = pkg_zalloc(sizeof(*index));
+  tmp = pkg_malloc(PATH_MAX);
+  index_path = pkg_malloc(PATH_MAX);
+  if (index == NULL || tmp == NULL || index_path == NULL)
+    {
+      pkg_error("unable to allocate index metadata buffer");
+      ret = -ENOMEM;
+      goto out;
+    }
+
+  /* Use a per-process FAT-compatible staging name. */
+
+  ret = snprintf(tmp, PATH_MAX, "%s/s%u.jsn", PKG_TMP_DIR,
+                 (unsigned int)getpid());
+  if (ret < 0 || (size_t)ret >= PATH_MAX)
+    {
+      pkg_error("temporary sync path is too long");
+      ret = -ENAMETOOLONG;
+      goto out;
+    }
+
+  ret = pkg_acquire_source(source, tmp);
+  if (ret < 0)
+    {
+      pkg_error("unable to fetch index source '%s': %d", source, ret);
+      goto out;
+    }
+
+  remove_tmp = true;
+  ret = pkg_metadata_load_index_path(tmp, index);
+  if (ret < 0)
+    {
+      pkg_error("downloaded index is invalid: %d", ret);
+      goto out;
+    }
+
+  ret = pkg_store_read_text(tmp, &text);
+  if (ret < 0)
+    {
+      pkg_error("unable to read fetched index: %d", ret);
+      goto out;
+    }
+
+  /* Commit the catalog and its source in one atomic file replacement. */
+
+  ret = pkg_repo_attach_source(&text, source);
+  if (ret < 0)
+    {
+      pkg_error("unable to record repository source: %d", ret);
+      goto out;
+    }
+
+  ret = pkg_store_format_index_path(index_path, PATH_MAX);
+  if (ret < 0)
+    {
+      pkg_error("unable to resolve local index path: %d", ret);
+      goto out;
+    }
+
+  ret = pkg_store_write_text_atomic(index_path, text);
+  if (ret < 0)
+    {
+      pkg_error("unable to write local index: %d", ret);
+      goto out;
+    }
+
+  pkg_info("synced package index from %s", source);
+  ret = 0;
+
+out:
+  if (remove_tmp)
+    {
+      pkg_store_remove_file(tmp);
+    }
+
+  pkg_free(text);
+  pkg_free(index);
+  pkg_free(tmp);
+  pkg_free(index_path);
+  pkg_lock_remove(lock);
+  pkg_free(lock);
+  return ret;
+}
+
+int pkg_available(FAR FILE *stream)
+{
+  FAR struct pkg_index_s *index;
+  FAR const char *arch;
+  FAR const char *compat;
+  size_t i;
+  int ret;
+
+  if (stream == NULL)
+    {
+      return EXIT_FAILURE;
+    }
+
+  index = pkg_zalloc(sizeof(*index));
+  if (index == NULL)
+    {
+      pkg_error("unable to allocate index metadata buffer");
+      return EXIT_FAILURE;
+    }
+
+  ret = pkg_store_prepare_layout();
+  if (ret < 0)
+    {
+      pkg_free(index);
+      pkg_error("unable to prepare package layout: %d", ret);
+      return EXIT_FAILURE;
+    }
+
+  ret = pkg_metadata_load_index(index);
+  if (ret < 0)
+    {
+      pkg_free(index);
+      pkg_error("unable to load package index: %d", ret);
+      return EXIT_FAILURE;
+    }
+
+  arch = pkg_runtime_arch();
+  compat = pkg_runtime_compat();
+
+  for (i = 0; i < index->count; i++)
+    {
+      FAR const struct pkg_manifest_s *manifest = &index->manifests[i];
+      FAR const struct pkg_manifest_s *latest;
+
+      if (strcmp(manifest->arch, arch) != 0 ||
+          strcmp(manifest->compat, compat) != 0)
+        {
+          continue;
+        }
+
+      latest = pkg_metadata_find_latest(index, manifest->name);
+      if (latest != manifest)
+        {
+          continue;
+        }
+
+      fprintf(stream,
+              "%s version=%s type=%s arch=%s compat=%s artifact=%s\n",
+              manifest->name,
+              manifest->version,
+              pkg_manifest_type_str(manifest->type),
+              manifest->arch,
+              manifest->compat,
+              manifest->artifact);
+    }
+
+  pkg_free(index);
+  return EXIT_SUCCESS;
+}