From 3d03526471f44f1eaedd6ac0baacb744f32e3636 Mon Sep 17 00:00:00 2001
From: Anton Nesterov <anton@demiurg.io>
Date: Fri, 10 Feb 2023 11:17:05 +0300
Subject: [PATCH] init

---
 .vscode/settings.json |   3 ++
 deps.ts               |  16 ++++++
 mod.ts                | 120 ++++++++++++++++++++++++++++++++++++++++++
 pick.ts               |   0
 readme.md             |   3 ++
 tests/.pick.yaml      |   0
 tests/mod_test.ts     |   1 +
 7 files changed, 143 insertions(+)
 create mode 100644 .vscode/settings.json
 create mode 100644 deps.ts
 create mode 100644 mod.ts
 create mode 100644 pick.ts
 create mode 100644 readme.md
 create mode 100644 tests/.pick.yaml
 create mode 100644 tests/mod_test.ts

diff --git a/.vscode/settings.json b/.vscode/settings.json
new file mode 100644
index 0000000..cbac569
--- /dev/null
+++ b/.vscode/settings.json
@@ -0,0 +1,3 @@
+{
+  "deno.enable": true
+}
diff --git a/deps.ts b/deps.ts
new file mode 100644
index 0000000..704d559
--- /dev/null
+++ b/deps.ts
@@ -0,0 +1,16 @@
+export {
+  type TarMeta,
+  Untar,
+} from "https://deno.land/std@0.144.0/archive/tar.ts";
+export {
+  copy,
+  readerFromStreamReader,
+} from "https://deno.land/std@0.144.0/streams/mod.ts";
+export { join } from "https://deno.land/std@0.144.0/path/mod.ts";
+export {
+  default as cacheDir,
+} from "https://deno.land/x/cache_dir@0.2.0/mod.ts";
+
+import * as Transform from "https://deno.land/x/transform@v0.4.0/mod.ts";
+export { Transform };
+export const { GzDecoder } = Transform.Transformers;
diff --git a/mod.ts b/mod.ts
new file mode 100644
index 0000000..ce54022
--- /dev/null
+++ b/mod.ts
@@ -0,0 +1,120 @@
+import {
+  cacheDir,
+  copy,
+  GzDecoder,
+  join,
+  readerFromStreamReader,
+  TarMeta,
+  Transform,
+  Untar,
+} from "./deps.ts";
+
+export const CACHE_DIR = join((await cacheDir()) ?? ".cache", "__pick_cache0");
+Deno.mkdir(CACHE_DIR).catch((_) => {});
+
+type TarEntry = TarMeta & Deno.Reader;
+interface GithubPickOptions {
+  repo: string;
+  version: string;
+  pick: RegExp[];
+}
+
+export async function* githubPick({ repo, version, pick }: GithubPickOptions) {
+  const _cached = await getFetchCache(`${repo}@${version}`);
+  if (_cached) {
+    const reader = await Deno.open(_cached);
+    yield* tarGzPickFiles(reader, pick);
+  } else {
+    yield* githubPickFiles({ repo, version, pick });
+  }
+}
+
+/**
+ * Reads *.tar.gz file from a version tag and returns generator of the files that match the pick regex.
+ *
+ * @param {GithubPickOptions} opts - { repo: "denoland/deno", version: "v1.0.0", pick: [/\.ts$/] }
+ * @returns {AsyncGenerator<TarEntry>}
+ */
+export async function* githubPickFiles(
+  { repo, version, pick }: GithubPickOptions,
+): AsyncGenerator<TarEntry> {
+  const targz = await fetch(
+    `https://github.com/${repo}/archive/refs/tags/${version}.tar.gz`,
+  );
+  await putFetchCache(targz.body, `${repo}@${version}`);
+  const reader = readerFromStreamReader(targz.body!.getReader());
+  yield* tarGzPickFiles(reader, pick);
+}
+
+/**
+ * Pick files from a tar.gz archive.
+ *
+ * @param {Deno.Reader} tar - Deno.open('archive.tar.gz')
+ * @param {RegExp[]} pick - [ /\.ts$/ ]
+ */
+export async function* tarGzPickFiles(
+  targz: Deno.Reader | Deno.FsFile,
+  pick: RegExp[],
+) {
+  const untar = new Untar(
+    Transform.newReader(
+      targz,
+      new GzDecoder(),
+    ),
+  );
+  for await (const entry of untar) {
+    if (entry.type === "file" && pick.some((re) => re.test(entry.fileName))) {
+      yield entry;
+    }
+  }
+}
+
+/**
+ * Writes fetch body to a cache file.
+ * @param body
+ * @param name
+ * @param opts
+ */
+export async function putFetchCache(body: Response["body"], name: string) {
+  const reader = readerFromStreamReader(body!.getReader());
+  const writer = await Deno.open(join(CACHE_DIR, name), { write: true });
+  await copy(reader, writer);
+  writer.close();
+}
+
+/**
+ * Get either the path to the cache file or null if it doesn't exist.
+ * @param {string} name
+ * @returns {Promise<string | null>}
+ */
+export async function getFetchCache(name: string) {
+  try {
+    const _file = join(CACHE_DIR, name);
+    await Deno.lstat(_file);
+    return _file;
+  } catch (_) {
+    return null;
+  }
+}
+
+/**
+ * Clean the fetch cache.
+ * @returns {Promise<void>}
+ */
+export async function cleanFetchCache() {
+  return Deno.remove(CACHE_DIR, { recursive: true });
+}
+
+export async function writeTarEntry(
+  entry: TarEntry,
+  dir: string,
+  formatWritePath: (path: string) => string = (path) => path,
+) {
+  const path = formatWritePath(entry.fileName);
+  const file = await Deno.open(dir + path, {
+    create: true,
+    write: true,
+  });
+  await copy(entry, file);
+  file.close();
+}
diff --git a/pick.ts b/pick.ts
new file mode 100644
index 0000000..e69de29
diff --git a/readme.md b/readme.md
new file mode 100644
index 0000000..923f653
--- /dev/null
+++ b/readme.md
@@ -0,0 +1,3 @@
+# Pick [WIP]
+
+Pick files from remote sources
\ No newline at end of file
diff --git a/tests/.pick.yaml b/tests/.pick.yaml
new file mode 100644
index 0000000..e69de29
diff --git a/tests/mod_test.ts b/tests/mod_test.ts
new file mode 100644
index 0000000..a191036
--- /dev/null
+++ b/tests/mod_test.ts
@@ -0,0 +1 @@
+const { test } = Deno;