Add include field to the manifest (#1842)

fixes #1770

---

**Stack**:
- #1846
- #1836
- #1842 ⬅


⚠️ *Part of a stack created by [spr](https://github.com/ejoffe/spr). Do
not merge manually using the UI - doing so may have unexpected results.*

scarb/src/core/manifest/mod.rs

@@ -75,6 +75,7 @@ pub struct ManifestMetadata {
     pub license_file: Option<Utf8PathBuf>,
     pub readme: Option<Utf8PathBuf>,
     pub repository: Option<String>,
+    pub include: Option<Vec<Utf8PathBuf>>,
     #[serde(rename = "tool")]
     pub tool_metadata: Option<BTreeMap<SmolStr, Value>>,
     pub cairo_version: Option<VersionReq>,

scarb/src/core/manifest/toml_manifest.rs

@@ -197,6 +197,7 @@ pub struct TomlPackage {
     pub license_file: Option<MaybeWorkspaceField<Utf8PathBuf>>,
     pub readme: Option<MaybeWorkspaceField<PathOrBool>>,
     pub repository: Option<MaybeWorkspaceField<String>>,
+    pub include: Option<Vec<Utf8PathBuf>>,
     /// **UNSTABLE** This package does not depend on Cairo's `core`.
     pub no_core: Option<bool>,
     pub cairo_version: Option<MaybeWorkspaceField<VersionReq>>,
@@ -571,6 +572,7 @@ impl TomlManifest {
                 .clone()
                 .map(|mw| mw.resolve("repository", || inheritable_package.repository()))
                 .transpose()?,
+            include: package.include.clone(),
             cairo_version: package
                 .cairo_version
                 .clone()

scarb/src/core/package/mod.rs

@@ -2,7 +2,7 @@ use std::fmt;
 use std::ops::Deref;
 use std::sync::Arc;
 
-use anyhow::{anyhow, Result};
+use anyhow::{anyhow, Context, Result};
 use camino::{Utf8Path, Utf8PathBuf};
 use serde::Deserialize;
 
@@ -12,6 +12,7 @@ use scarb_ui::args::WithManifestPath;
 
 use crate::core::manifest::Manifest;
 use crate::core::{Target, TargetKind};
+use crate::internal::fsx;
 
 mod id;
 mod name;
@@ -105,6 +106,26 @@ impl Package {
             .get(tool_name)
     }
 
+    pub fn include(&self) -> Result<Vec<Utf8PathBuf>> {
+        self.manifest
+            .as_ref()
+            .metadata
+            .include
+            .as_ref()
+            .map(|include| {
+                include
+                    .iter()
+                    .map(|path| {
+                        let path = self.root().join(path);
+                        let path = fsx::canonicalize_utf8(&path)
+                            .with_context(|| format!("failed to find included file at {path}"))?;
+                        Ok(path)
+                    })
+                    .collect::<Result<Vec<_>>>()
+            })
+            .unwrap_or_else(|| Ok(Vec::new()))
+    }
+
     pub fn fetch_tool_metadata(&self, tool_name: &str) -> Result<&toml::Value> {
         self.tool_metadata(tool_name)
             .ok_or_else(|| anyhow!("package manifest `{self}` has no [tool.{tool_name}] section"))

scarb/src/core/publishing/manifest_normalization.rs

@@ -1,9 +1,5 @@
 use std::collections::BTreeMap;
 
-use anyhow::{bail, Result};
-use camino::Utf8PathBuf;
-use indoc::formatdoc;
-
 use crate::core::{TomlCairoPluginTargetParams, TomlTarget};
 use crate::{
     core::{
@@ -13,6 +9,10 @@ use crate::{
     },
     DEFAULT_LICENSE_FILE_NAME, DEFAULT_README_FILE_NAME,
 };
+use anyhow::{bail, Result};
+use camino::Utf8PathBuf;
+use indoc::formatdoc;
+use itertools::Itertools;
 
 pub fn prepare_manifest_for_publish(pkg: &Package) -> Result<TomlManifest> {
     let package = Some(generate_package(pkg));
@@ -73,6 +73,10 @@ fn generate_package(pkg: &Package) -> Box<TomlPackage> {
             .clone()
             .map(|_| MaybeWorkspace::Defined((Utf8PathBuf::from(DEFAULT_README_FILE_NAME)).into())),
         repository: metadata.repository.clone().map(MaybeWorkspace::Defined),
+        include: metadata.include.as_ref().map(|x| {
+            // Sort for stability.
+            x.iter().sorted().cloned().collect_vec()
+        }),
         no_core: summary.no_core.then_some(true),
         cairo_version: metadata.cairo_version.clone().map(MaybeWorkspace::Defined),
         experimental_features: pkg.manifest.experimental_features.clone(),

scarb/src/core/publishing/source.rs

@@ -77,8 +77,11 @@ fn push_worktree_files(pkg: &Package, ret: &mut Vec<Utf8PathBuf>) -> Result<()>
             true
         }
     };
-
-    WalkBuilder::new(pkg.root())
+    let mut builder = WalkBuilder::new(pkg.root());
+    for path in pkg.include()? {
+        builder.add(&path);
+    }
+    builder
         .follow_links(true)
         .standard_filters(true)
         .parents(false)

scarb/src/ops/package.rs

@@ -2,7 +2,7 @@ use std::collections::BTreeMap;
 use std::fs::File;
 use std::io::{Seek, SeekFrom, Write};
 
-use anyhow::{bail, ensure, Context, Result};
+use anyhow::{anyhow, bail, ensure, Context, Result};
 use camino::Utf8PathBuf;
 use indoc::{formatdoc, indoc, writedoc};
 
@@ -402,7 +402,10 @@ fn source_files(pkg: &Package) -> Result<ArchiveRecipe> {
     list_source_files(pkg)?
         .into_iter()
         .map(|on_disk| {
-            let path = on_disk.strip_prefix(pkg.root())?.to_owned();
+            let path = on_disk
+                .strip_prefix(pkg.root())
+                .map_err(|_| anyhow!("file `{on_disk}` is not part of `{}`", pkg.id.name))?
+                .to_owned();
             Ok(ArchiveFile {
                 path,
                 contents: ArchiveFileContents::OnDisk(on_disk),

scarb/tests/package.rs

@@ -1525,3 +1525,154 @@ fn package_with_publish_disabled() {
             [..]Packaged [..] files, [..] ([..] compressed)
         "#});
 }
+
+#[test]
+fn can_include_additional_files() {
+    let t = TempDir::new().unwrap();
+    simple_project()
+        .manifest_package_extra(indoc! {r#"
+            include = ["target/file.txt", "target/some/", "other/file.txt", "other/some"]
+        "#})
+        .build(&t);
+
+    t.child("target/file.txt")
+        .write_str("some file content")
+        .unwrap();
+    t.child("target/some/file.txt")
+        .write_str("some file content")
+        .unwrap();
+    t.child("other/file.txt")
+        .write_str("some file content")
+        .unwrap();
+    t.child("other/some/dir/file.txt")
+        .write_str("some file content")
+        .unwrap();
+    t.child("other/some/dir/other.txt")
+        .write_str("some file content")
+        .unwrap();
+
+    t.child(".gitignore").write_str("target").unwrap();
+    t.child(".scarbignore").write_str("other").unwrap();
+
+    Scarb::quick_snapbox()
+        .arg("package")
+        .arg("--no-metadata")
+        .current_dir(&t)
+        .assert()
+        .success()
+        .stdout_matches(indoc! {r#"
+        [..] Packaging foo v1.0.0 [..]
+        [..] Verifying foo-1.0.0.tar.zst
+        [..] Compiling foo v1.0.0 ([..])
+        [..]  Finished `dev` profile target(s) in [..]
+        [..]  Packaged [..] files, [..] ([..] compressed)
+        "#});
+
+    PackageChecker::assert(&t.child("target/package/foo-1.0.0.tar.zst"))
+        .name_and_version("foo", "1.0.0")
+        .contents(&[
+            "VERSION",
+            "Scarb.orig.toml",
+            "Scarb.toml",
+            "src/lib.cairo",
+            "src/foo.cairo",
+            "other/some/dir/other.txt",
+            "other/some/dir/file.txt",
+            "other/file.txt",
+            "target/some/file.txt",
+            "target/file.txt",
+        ])
+        .file_eq("VERSION", "1")
+        .file_eq_path("Scarb.orig.toml", t.child("Scarb.toml"))
+        .file_eq_path("src/lib.cairo", t.child("src/lib.cairo"))
+        .file_eq_path("src/foo.cairo", t.child("src/foo.cairo"))
+        .file_eq_path(
+            "other/some/dir/other.txt",
+            t.child("other/some/dir/other.txt"),
+        )
+        .file_eq_path(
+            "other/some/dir/file.txt",
+            t.child("other/some/dir/file.txt"),
+        )
+        .file_eq_path("other/file.txt", t.child("other/file.txt"))
+        .file_eq_path("target/some/file.txt", t.child("target/some/file.txt"))
+        .file_eq_path("target/file.txt", t.child("target/file.txt"))
+        .file_matches_nl(
+            "Scarb.toml",
+            indoc! {r#"
+            # Code generated by scarb package -p foo; DO NOT EDIT.
+            #
+            # When uploading packages to the registry Scarb will automatically
+            # "normalize" Scarb.toml files for maximal compatibility
+            # with all versions of Scarb and also rewrite `path` dependencies
+            # to registry dependencies.
+            #
+            # If you are reading this file be aware that the original Scarb.toml
+            # will likely look very different (and much more reasonable).
+            # See Scarb.orig.toml for the original contents.
+    
+            [package]
+            name = "foo"
+            version = "1.0.0"
+            edition = "2023_01"
+            include = [
+                "other/file.txt",
+                "other/some",
+                "target/file.txt",
+                "target/some/",
+            ]
+
+            [dependencies]
+        "#},
+        );
+}
+
+#[test]
+fn files_outside_package_cannot_be_included() {
+    let t = TempDir::new().unwrap();
+    let pkg = t.child("pkg");
+    simple_project()
+        .manifest_package_extra(indoc! {r#"
+            include = ["../some/file.txt"]
+        "#})
+        .build(&pkg);
+    t.child("some/file.txt")
+        .write_str("some file content")
+        .unwrap();
+    Scarb::quick_snapbox()
+        .arg("package")
+        .arg("--no-metadata")
+        .current_dir(&pkg)
+        .assert()
+        .failure()
+        .stdout_matches(indoc! {r#"
+        [..] Packaging foo v1.0.0 [..]
+        error: file `[..]file.txt` is not part of `foo`
+        "#});
+}
+
+#[test]
+fn files_that_dont_exist_during_packaging_cannot_be_included() {
+    let t = TempDir::new().unwrap();
+    let pkg = t.child("pkg");
+    simple_project()
+        .manifest_package_extra(indoc! {r#"
+            include = ["some/file.txt"]
+        "#})
+        .build(&pkg);
+    Scarb::quick_snapbox()
+        .arg("package")
+        .arg("--no-metadata")
+        .current_dir(&pkg)
+        .assert()
+        .failure()
+        .stdout_matches(indoc! {r#"
+        [..] Packaging foo v1.0.0 [..]
+        error: failed to list source files in: [..]pkg
+
+        Caused by:
+            0: failed to find included file at [..]file.txt
+            1: failed to get absolute path of `[..]file.txt`
+            2: [..]
+        "#});
+}

website/docs/reference/manifest.md

@@ -100,6 +100,20 @@ Setting the `cairo-version` key in `[package]` will affect all targets in the pa
 The value in this field will not affect the version of the compiler run by Scarb.
 Scarb always uses its built-in version of the Cairo compiler.
 
+### `include`
+
+When packaging a package with `scarb package` command (see
+[packaging your package](../registries/publishing.md#packaging-your-package)), all files excluded with rules from
+`.gitignore` or `.scarbignore` files are not included in the resulting package tarball.
+This field can be used mark files and subdirectories that should be included in the package tarball, even if those files
+would be excluded by rules from ignore files.
+The paths are relative to the package root and cannot point to files outside the package.
+
+```toml
+[package]
+include = ["target/some/file.txt"]
+```
+
 ### `authors`
 
 This optional field lists the people or organizations that are considered the "authors" of the package.

website/docs/registries/publishing.md

@@ -7,9 +7,11 @@ Once uploaded, it will be available for other users to download and use.
 
 To upload your package, use the scarb publish command.
 By default, this command will publish your package to the official [scarbs.xyz](https://scarbs.xyz) registry.
-The publish command automatically [packages and verifies](#packaging-your-package) your package, so there is no need to run `scarb package` beforehand.
+The publish command automatically [packages and verifies](#packaging-your-package) your package, so there is no need to
+run `scarb package` beforehand.
 
-To publish your package to a registry that supports package publishing, you need to authenticate using an API token with the `publish` scope.
+To publish your package to a registry that supports package publishing, you need to authenticate using an API token with
+the `publish` scope.
 First, log in to the registry and [in the dashboard](https://scarbs.xyz/dashboard) generate the API token.
 Scarb will use the token to authenticate and complete the publishing process.
 The token must be provided via the `SCARB_REGISTRY_AUTH_TOKEN` environment variable.
@@ -44,17 +46,35 @@ publish = false
 
 Use the `scarb package` command to create an archive of your package.
 You can read about the package compression algorithm and contents in the [Package tarball](./package-tarball) section.
-Basically when you run the command, Scarb gathers the source code of your package along with metadata files, such as the manifest file, and places them in an archive in `target/package` directory.
+Basically when you run the command, Scarb gathers the source code of your package along with metadata files, such as the
+manifest file, and places them in an archive in `target/package` directory.
 
-If you are in a Git repository, Scarb will first check if the repo state is clean and error out in case of any changes present in the Git working directory.
+If you are in a Git repository, Scarb will first check if the repo state is clean and error out in case of any changes
+present in the Git working directory.
 To bypass this check, you can use the `--allow-dirty` flag.
 
 The next step is package verification.
-After creating the initial archive, Scarb will attempt to unpack it and compile to check for any corruptions in the packaging process.
+After creating the initial archive, Scarb will attempt to unpack it and compile to check for any corruptions in the
+packaging process.
 If you want to speed up the packaging process, you can disable this step using the `--no-verify` flag.
 
 > [!WARNING]
 > This is a dangerous operation as it can lead to uploading a corrupted package to the registry.
 > Please use with caution.
 
-After successfully completing the whole process, the `{name}-{version}.tar.zst` archive waits in the `target/package` directory for being uploaded, where both `name` and `version` correspond to the values in `Scarb.toml`.
+After successfully completing the whole process, the `{name}-{version}.tar.zst` archive waits in the `target/package`
+directory for being uploaded, where both `name` and `version` correspond to the values in `Scarb.toml`.
+
+### Files included in the package
+
+All files in the package directory are included in the resulting tarball, except for the following:
+
+- Files excluded with rules defined in any `.scarbignore`, `.gitignore` or `.ignore` files.
+- The `<package root>/target` directory.
+- Any subdirectories containing `Scarb.toml` file.
+- The `.git` directory.
+- Symlinks within the package directory are followed, while symlinks outside are ignored.
+- File system boundaries are not crossed.
+
+Files that would be otherwise ignored by the rules listed above, can still be included
+with [include](../reference/manifest.md#include) field.