add tests and command line interface

.editorconfig

@@ -0,0 +1,7 @@
+[*]
+end_of_line = lf
+insert_final_newline = true
+indent_style = space
+indent_size = 2
+charset = utf-8
+trim_trailing_whitespace = true

deno.lock

@@ -40,6 +40,7 @@
     "https://deno.land/x/transform@v0.4.0/transformers/zlib/gzip.ts": "8008eb620908025c9768bf2fe43c542b122224deb13ce37034237ed44fa97827",
     "https://deno.land/x/transform@v0.4.0/transformers/zlib/mod.ts": "ed83911768d491342fb3e09a9b23217c9345ce0b721c69346ed53416d203e195",
     "https://deno.land/x/transform@v0.4.0/transformers/zlib/pako.js": "f7077bb75f3f8a479fc945cffa93faa113be1d6c1d1c6d8566d35a6406ad40b8",
+    "https://deno.land/x/yaml@v2.2.1/util.js": "2f1db26f6cc426ef698210b592f40cec49be1b6c4b34e7f2d61904786242bd85",
     "https://raw.githubusercontent.com/marcosc90/encoding-wasm/v0.1.0/dist/encoding_wasm.js": "d7bb9e589b3505f75d220eadb127d98a3b0ef19a884bafc8f94db7a90a24eacf",
     "https://raw.githubusercontent.com/marcosc90/encoding-wasm/v0.1.0/dist/encoding_wasm_bytes.js": "c826533d17099db2112c9ddb3cbcd013f7afef5273f4666ac7e0196cc6424659",
     "https://raw.githubusercontent.com/marcosc90/encoding-wasm/v0.1.0/mod.ts": "0ebef5c2b134fa6b80ebd52242b23cd19b0dd8d661d07d3fb6cef7b971da99f7"

deps.ts

@@ -1,12 +1,9 @@
-export {
+export { Untar } from "https://deno.land/std@0.144.0/archive/tar.ts";
-  type TarMeta,
-  Untar,
-} from "https://deno.land/std@0.144.0/archive/tar.ts";
 export {
   copy,
   readerFromStreamReader,
 } from "https://deno.land/std@0.144.0/streams/mod.ts";
-export { join } from "https://deno.land/std@0.144.0/path/mod.ts";
+export { globToRegExp, join } from "https://deno.land/std@0.144.0/path/mod.ts";
 export {
   default as cacheDir,
 } from "https://deno.land/x/cache_dir@0.2.0/mod.ts";

mod.d.ts

@@ -0,0 +1,34 @@
+export type ReadStrategy =
+  | "tar"
+  | "targz"
+  | "github";
+
+export type ReadPredicate = (
+  entry: ConfigEntry,
+) => AsyncGenerator<Deno.Reader | Deno.Reader & FileMeta>;
+
+export type GlobLike = string | RegExp;
+export interface ConfigEntry {
+  strategy?: ReadStrategy;
+  read?: ReadPredicate;
+  source: string;
+  output: string;
+  pick: GlobLike[];
+}
+
+export type PickConfig = ConfigEntry[];
+
+export interface FileMeta extends Deno.Reader {
+  fileName?: string;
+}
+
+export interface ReadableEntry extends ConfigEntry {
+  read: () => AsyncGenerator<FileMeta>;
+}
+
+export type PathLike = string | URL;
+export interface GithubPickOptions {
+  repo: string;
+  version: string;
+  pick: RegExp[];
+}

mod.ts

@@ -1,29 +1,118 @@
 import {
   cacheDir,
   copy,
+  globToRegExp,
   GzDecoder,
   join,
   readerFromStreamReader,
-  TarMeta,
   Transform,
   Untar,
 } from "./deps.ts";
+import type {
+  ConfigEntry,
+  GithubPickOptions,
+  PathLike,
+  PickConfig,
+  ReadableEntry,
+  ReadPredicate,
+  ReadStrategy,
+} from "./mod.d.ts";
 
 export const CACHE_DIR = join((await cacheDir()) ?? ".cache", "__pick_cache0");
 await Deno.mkdir(CACHE_DIR).catch((_) => {});
 
-type TarEntry = TarMeta & Deno.Reader;
+/**
-interface GithubPickOptions {
+ * Read config and write files
-  repo: string;
+ */
-  version: string;
+export async function write(entires: PickConfig) {
-  pick: RegExp[];
+  const _entries: ReadableEntry[] = entires.map(asReadable$);
+  for (const entry of _entries) {
+    const output = entry.output;
+    await Deno.mkdir(output, { recursive: true });
+    for await (const reader of entry.read()) {
+      const file = await Deno.open(
+        join(output, filename(reader.fileName ?? "")),
+        { create: true, write: true },
+      );
+      await copy(reader, file);
+      file.close();
+    }
+  }
+}
+
+/**
+ * Decorator which makes config entries readable
+ */
+function asReadable$(entry: ConfigEntry): ReadableEntry {
+  // convert globs to regex if required
+  const pick = entry.pick.map((p) => {
+    if (typeof p === "string") {
+      return globToRegExp(p);
+    }
+    return p;
+  });
+
+  // if read is a function, use it
+  let read: ReadableEntry["read"];
+  if (typeof entry.read === "function") {
+    read = () => (entry.read as ReadPredicate)(entry);
+  } else {
+    // otherwise, use a specific strategy
+    const strategy = entry.strategy ?? autoStrategy$(entry.source);
+    switch (strategy) {
+      case "tar": {
+        read = () => tarPickFiles(entry.source, pick);
+        break;
+      }
+      case "targz": {
+        read = () => tarGzPickFiles(entry.source, pick);
+        break;
+      }
+      case "github": {
+        const [repo, version] = entry.source.split("@");
+        if (!version) {
+          throw new Error(
+            `Invalid source format: ${entry.source} 
+            Expected: <repo>@<version>`,
+          );
+        }
+        read = () => githubPick({ repo, version, pick });
+        break;
+      }
+      default: {
+        throw new Error(`Unknown strategy: ${entry.strategy}`);
+      }
+    }
+  }
+  return {
+    ...entry,
+    read,
+  };
+}
+
+function autoStrategy$(source: string): ReadStrategy {
+  if (source.endsWith(".tar")) {
+    return "tar";
+  } else if (source.endsWith(".tar.gz")) {
+    return "targz";
+  } else if (source.includes("@")) {
+    return "github";
+  } else {
+    throw new Error(`Unknown strategy for source: ${source}`);
+  }
+}
+
+function filename(path: string) {
+  return path.split("/").pop() ?? "";
 }
 
 /**
  * Pick files from a github repo.
  * @param {GithubPickOptions} opts - { repo: "denoland/deno", version: "v1.0.0", pick: [/\.ts$/] }
  */
-export async function* githubPick({ repo, version, pick }: GithubPickOptions) {
+export async function* githubPick(
+  { repo, version, pick }: GithubPickOptions,
+): ReturnType<ReadableEntry["read"]> {
   await Deno.mkdir(join(CACHE_DIR, repo.split("/").shift() ?? "")).catch(
     (_) => {},
   );
@@ -41,11 +130,10 @@ export async function* githubPick({ repo, version, pick }: GithubPickOptions) {
  * Reads *.tar.gz file from a version tag and returns generator of the files that match the pick regex.
  *
  * @param {GithubPickOptions} opts - { repo: "denoland/deno", version: "v1.0.0", pick: [/\.ts$/] }
- * @returns {AsyncGenerator<TarEntry>}
  */
 export async function* githubPickFiles(
   { repo, version, pick }: GithubPickOptions,
-): AsyncGenerator<TarEntry> {
+): ReturnType<ReadableEntry["read"]> {
   const targz = await fetch(
     `https://github.com/${repo}/archive/refs/tags/${version}.tar.gz`,
   );
@@ -59,13 +147,16 @@ export async function* githubPickFiles(
 /**
  * Pick files from a tar.gz archive.
  *
- * @param {Deno.Reader} tar - Deno.open('archive.tar.gz')
+ * @param {Deno.Reader} targz - Deno.open('archive.tar.gz')
  * @param {RegExp[]} pick - [ /\.ts$/ ]
  */
 export async function* tarGzPickFiles(
-  targz: Deno.Reader | Deno.FsFile,
+  targz: Deno.Reader | Deno.FsFile | PathLike,
   pick: RegExp[],
-) {
+): ReturnType<ReadableEntry["read"]> {
+  if (typeof targz === "string" || targz instanceof URL) {
+    targz = await Deno.open(targz);
+  }
   const untar = new Untar(
     Transform.newReader(
       targz,
@@ -79,15 +170,35 @@ export async function* tarGzPickFiles(
   }
 }
 
+/**
+ * Pick files from a tar.gz archive.
+ *
+ * @param {Deno.Reader} tar - Deno.open('archive.tar')
+ * @param {RegExp[]} pick - [ /\.ts$/ ]
+ */
+export async function* tarPickFiles(
+  tar: Deno.Reader | Deno.FsFile | PathLike,
+  pick: RegExp[],
+): ReturnType<ReadableEntry["read"]> {
+  if (typeof tar === "string" || tar instanceof URL) {
+    tar = await Deno.open(tar);
+  }
+  const untar = new Untar(tar);
+  for await (const entry of untar) {
+    if (entry.type === "file" && pick.some((re) => re.test(entry.fileName))) {
+      yield entry;
+    }
+  }
+}
+
 /**
  * Writes fetch body to a cache file.
- * @param body
- * @param name
- * @param opts
  */
 export async function putFetchCache(body: Response["body"], name: string) {
   const reader = readerFromStreamReader(body!.getReader());
-  const writer = await Deno.open(join(CACHE_DIR, name), {
+  const _file = join(CACHE_DIR, name);
+  await Deno.remove(_file).catch((_) => {});
+  const writer = await Deno.open(_file, {
     write: true,
     create: true,
   });
@@ -97,8 +208,6 @@ export async function putFetchCache(body: Response["body"], name: string) {
 
 /**
  * Get either the path to the cache file or null if it doesn't exist.
- * @param {string} name
- * @returns {Promise<string | null>}
  */
 export async function getFetchCache(name: string) {
   try {
@@ -112,22 +221,7 @@ export async function getFetchCache(name: string) {
 
 /**
  * Clean the fetch cache.
- * @returns {Promise<void>}
  */
 export async function cleanFetchCache() {
-  return Deno.remove(CACHE_DIR, { recursive: true });
+  return await Deno.remove(CACHE_DIR, { recursive: true });
-}
-
-export async function writeTarEntry(
-  entry: TarEntry,
-  dir: string,
-  formatWritePath: (path: string) => string = (path) => path,
-) {
-  const path = formatWritePath(entry.fileName);
-  const file = await Deno.open(dir + path, {
-    create: true,
-    write: true,
-  });
-  await copy(entry, file);
-  file.close();
 }

pickit.ts

@@ -0,0 +1,81 @@
+// command line interface for pickit
+// semantics:
+//   - pickit [source] [outputDir] [glob1] [glob2]...
+//   - pickit ./config.ts (default: ./.pickit.ts)
+
+import { cleanFetchCache, write } from "./mod.ts";
+
+if (Deno.args.length == 1) {
+  if (Deno.args[0] == "clean") {
+    await cleanFetchCache();
+  } else {
+    const config = await import(Deno.args[0]);
+    await write(config.default);
+  }
+} else if (Deno.args.length >= 3) {
+  const [source, output, ...pick] = Deno.args;
+  await write([{ source, output, pick }]);
+} else {
+  console.log(
+    `
+  %cPickIt
+  `,
+    "font-size: 1.5em; font-weight: bold",
+  );
+  console.log(`
+        This utility helps you to extract files from tarballs and github repos using glob syntax or regular expressions.
+        You can use either a config file or command line arguments.
+    `);
+  console.log(
+    `
+  %cUsage:
+  `,
+    "font-size: 1.5em; font-weight: bold",
+  );
+  console.log(
+    `
+        pickit %c[source] [outputDir] [glob1] [glob2]...
+        %cpickit %c./config.ts
+    `,
+    "color: #ccb;",
+    "",
+    "color: #ccb;",
+  );
+  console.log(
+    `
+  %cExample:
+  `,
+    "font-size: 1.5em; font-weight: bold",
+  );
+  console.log(
+    `
+        %cpickit nesterow/pickit@v0.0.1 scripts *.d.ts **/tests/*.ts
+    `,
+    "color: #aab;",
+  );
+  console.log(
+    `
+  %cConfiguring (ts):
+  `,
+    "font-size: 1.5em; font-weight: bold",
+  );
+  console.log(
+    `%c
+        import type { PickConfig } from "https://deno.land/x/pickit/mod.d.ts";
+        export default [
+            {
+                source: "username/repo@version",
+                output: "./outputDir",
+                pick: [
+                    /^.*\/base\/.*\.css$/,
+                    "/src/index.js",
+                    "/src/**/*.yaml"
+                ],
+            },
+        ] as PickConfig;
+
+        
+    `,
+    "color: #aab;",
+  );
+}

tests/config_mock.ts

@@ -0,0 +1,11 @@
+import type { PickConfig } from "../mod.d.ts";
+
+export default [
+  {
+    source: "saadeghi/daisyui@v2.47.0",
+    output: ".basestyles",
+    pick: [
+      /^.*\/base\/.*\.css$/,
+    ],
+  },
+] as PickConfig;

tests/mod_test.ts

@@ -1,28 +1,93 @@
 import { assert } from "https://deno.land/std@0.144.0/_util/assert.ts";
-import {join} from '../deps.ts';
+import { join } from "../deps.ts";
 import * as mod from "../mod.ts";
+import type { FileMeta, PickConfig } from "../mod.d.ts";
 const { test } = Deno;
 
 test("mod", async (t) => {
   const repo = "saadeghi/daisyui";
   const version = "v2.47.0";
-  await t.step("githubPick", async () => {
+
+  await t.step("mod.githubPick", async () => {
     const pick = [
       /^.*\/base\/.*\.css$/,
       /src\/index\.js/,
     ];
-    const files = [];
+    const files: FileMeta[] = [];
     for await (const file of mod.githubPick({ repo, version, pick })) {
       files.push(file);
     }
     assert(files.length > 0);
-    assert(files.filter(f => f.fileName.endsWith(".js")).length  == 1);
+    assert(files.filter((f) => f.fileName?.endsWith(".js")).length == 1);
-    assert(files.filter(f => f.fileName.endsWith(".css")).length  == 2);
+    assert(files.filter((f) => f.fileName?.endsWith(".css")).length == 2);
   });
-  await t.step("is cached", async () => {
+
+  await t.step("mod.getFetchCache", async () => {
     const name = `${repo}@${version}`;
     const cached = await mod.getFetchCache(name);
     assert(cached?.endsWith(version));
     assert(cached == join(mod.CACHE_DIR, name));
   });
+
+  await t.step("mod.write: autoStrategy", async () => {
+    const output = ".daisyui";
+    const config: PickConfig = [
+      {
+        source: `${repo}@${version}`,
+        output,
+        pick: [
+          /^.*\/base\/.*\.css$/,
+          /src\/index\.js/,
+        ],
+      },
+    ];
+    await mod.write(config);
+    const files = await Deno.readDir(output);
+    for await (const file of files) {
+      assert(file.isFile);
+      assert(file.name.endsWith(".js") || file.name.endsWith(".css"));
+      await Deno.remove(join(output, file.name));
+    }
+    await Deno.remove(output);
+  });
+
+  await t.step("mod.write: use glob", async () => {
+    const output = ".daisyui";
+    const config: PickConfig = [
+      {
+        source: `${repo}@${version}`,
+        output,
+        pick: [
+          "**/base/*.css",
+          "src/index.js",
+        ],
+      },
+    ];
+    await mod.write(config);
+    const files = await Deno.readDir(output);
+    for await (const file of files) {
+      assert(file.isFile);
+      assert(file.name.endsWith(".js") || file.name.endsWith(".css"));
+      await Deno.remove(join(output, file.name));
+    }
+    await Deno.remove(output);
+  });
+});
+
+test("pickit: cmd line interface", async () => {
+  const p = await Deno.run({
+    cmd: [
+      "deno",
+      "run",
+      "-A",
+      "pickit.ts",
+      "saadeghi/daisyui@v2.47.0",
+      "daisyui",
+      "**/src/index.js",
+      "**/base/*.css",
+    ],
+  });
+  const status = await p.status();
+  assert(status.success);
+  await Deno.remove("daisyui", { recursive: true });
 });