From a4f67c1495303e28d45a37278c5533f3d9f14295 Mon Sep 17 00:00:00 2001 From: Anton Nesterov Date: Fri, 10 Feb 2023 19:30:37 +0300 Subject: [PATCH] pick files from tar.gz and github --- deno.json | 8 ++++++++ deno.lock | 47 +++++++++++++++++++++++++++++++++++++++++++++++ mod.ts | 23 ++++++++++++++++++----- readme.md | 2 +- tests/mod_test.ts | 27 +++++++++++++++++++++++++++ 5 files changed, 101 insertions(+), 6 deletions(-) create mode 100644 deno.json create mode 100644 deno.lock diff --git a/deno.json b/deno.json new file mode 100644 index 0000000..04d9ebe --- /dev/null +++ b/deno.json @@ -0,0 +1,8 @@ +{ + "tasks": { + "test": "deno test -A --no-check tests/" + }, + "compilerOptions": { + "strict": true + } +} diff --git a/deno.lock b/deno.lock new file mode 100644 index 0000000..70f0fef --- /dev/null +++ b/deno.lock @@ -0,0 +1,47 @@ +{ + "version": "2", + "remote": { + "https://deno.land/std@0.144.0/_util/assert.ts": "e94f2eb37cebd7f199952e242c77654e43333c1ac4c5c700e929ea3aa5489f74", + "https://deno.land/std@0.144.0/_util/os.ts": "3b4c6e27febd119d36a416d7a97bd3b0251b77c88942c8f16ee5953ea13e2e49", + "https://deno.land/std@0.144.0/archive/tar.ts": "ea420312bb9105d5c960133563d0eacf0f2f86e097e847cd84c2172c687a1125", + "https://deno.land/std@0.144.0/async/deferred.ts": "bc18e28108252c9f67dfca2bbc4587c3cbf3aeb6e155f8c864ca8ecff992b98a", + "https://deno.land/std@0.144.0/bytes/bytes_list.ts": "aba5e2369e77d426b10af1de0dcc4531acecec27f9b9056f4f7bfbf8ac147ab4", + "https://deno.land/std@0.144.0/bytes/equals.ts": "3c3558c3ae85526f84510aa2b48ab2ad7bdd899e2e0f5b7a8ffc85acb3a6043a", + "https://deno.land/std@0.144.0/bytes/mod.ts": "763f97d33051cc3f28af1a688dfe2830841192a9fea0cbaa55f927b49d49d0bf", + "https://deno.land/std@0.144.0/io/buffer.ts": "bd0c4bf53db4b4be916ca5963e454bddfd3fcd45039041ea161dbf826817822b", + "https://deno.land/std@0.144.0/io/readers.ts": "45847ad404afd2f605eae1cff193f223462bc55eeb9ae313c2f3db28aada0fd6", + "https://deno.land/std@0.144.0/io/types.d.ts": "0cae3a62da7a37043661746c65c021058bae020b54e50c0e774916e5d4baee43", + "https://deno.land/std@0.144.0/path/_constants.ts": "df1db3ffa6dd6d1252cc9617e5d72165cd2483df90e93833e13580687b6083c3", + "https://deno.land/std@0.144.0/path/_interface.ts": "ee3b431a336b80cf445441109d089b70d87d5e248f4f90ff906820889ecf8d09", + "https://deno.land/std@0.144.0/path/_util.ts": "c1e9686d0164e29f7d880b2158971d805b6e0efc3110d0b3e24e4b8af2190d2b", + "https://deno.land/std@0.144.0/path/common.ts": "bee563630abd2d97f99d83c96c2fa0cca7cee103e8cb4e7699ec4d5db7bd2633", + "https://deno.land/std@0.144.0/path/glob.ts": "cb5255638de1048973c3e69e420c77dc04f75755524cb3b2e160fe9277d939ee", + "https://deno.land/std@0.144.0/path/mod.ts": "4945b430b759b0b3d98f2a278542cbcf95e0ad2bd8eaaed3c67322b306b2b346", + "https://deno.land/std@0.144.0/path/posix.ts": "293cdaec3ecccec0a9cc2b534302dfe308adb6f10861fa183275d6695faace44", + "https://deno.land/std@0.144.0/path/separator.ts": "fe1816cb765a8068afb3e8f13ad272351c85cbc739af56dacfc7d93d710fe0f9", + "https://deno.land/std@0.144.0/path/win32.ts": "31811536855e19ba37a999cd8d1b62078235548d67902ece4aa6b814596dd757", + "https://deno.land/std@0.144.0/streams/buffer.ts": "ee47194022d47fa23d4749b8afbadc83c237c4595467a9bddb322af0dd205815", + "https://deno.land/std@0.144.0/streams/conversion.ts": "fc3db02026183da795fa32ac7549868e9f19c75ba029d4b4c3739af62b48517a", + "https://deno.land/std@0.144.0/streams/delimiter.ts": "81574a35e433d3b392887118d02fce12fec611fe3ff6b8f3d3758d141b7201e9", + "https://deno.land/std@0.144.0/streams/merge.ts": "88ed3dfa030ae076802688e4cadd762a251a41d81ed1776dfd9a2a9a0e970195", + "https://deno.land/std@0.144.0/streams/mod.ts": "d7d8fa97f79bca052eba8b6df9c4602403d144cf97ec9f6d0896c787dae0a9de", + "https://deno.land/std@0.57.0/async/deferred.ts": "ac95025f46580cf5197928ba90995d87f26e202c19ad961bc4e3177310894cdc", + "https://deno.land/x/cache_dir@0.2.0/mod.ts": "89ac1bc4d8f1c52fde192e0470d0597fa156ecf2b53d7fe162a11fd365c79920", + "https://deno.land/x/transform@v0.2.0/transform.ts": "2d3690dae85be1ec7a6e09025ef651e0b683910f7470b474adf59406df917da6", + "https://deno.land/x/transform@v0.4.0/mod.ts": "6e039b9e4bdbc0cb57deb48c955f5e26fe7251b7619ea7d79105f95487683838", + "https://deno.land/x/transform@v0.4.0/transform.ts": "2d3690dae85be1ec7a6e09025ef651e0b683910f7470b474adf59406df917da6", + "https://deno.land/x/transform@v0.4.0/transformers/base64/decoder.ts": "7d533dd86cfbb097a89eb83d6e9539a3330b631f11975d8774716c234178793a", + "https://deno.land/x/transform@v0.4.0/transformers/base64/deps.ts": "f56c87286aee9b254ed008219dacabd701a559d5c0f8eb7ac7bafcf3ffe480d3", + "https://deno.land/x/transform@v0.4.0/transformers/base64/encoder.ts": "9077a79488ea9bdf34203c5056ea6c225eda38503282fed26746b26d5d2779e9", + "https://deno.land/x/transform@v0.4.0/transformers/base64/mod.ts": "6b555deb6b31319f043106c6816aa03d80b3965f9badad1befb10ee303b764b4", + "https://deno.land/x/transform@v0.4.0/transformers/base64/util.ts": "cd9f13c3a03c43c34e605c65dd1e0655711ca9fbbf6c4663ecf0213009bb96c4", + "https://deno.land/x/transform@v0.4.0/transformers/errors.ts": "958ddc017ed11067ffff470acdae4c6107c2de3a20fa2df28c2ee3586596d647", + "https://deno.land/x/transform@v0.4.0/transformers/mod.ts": "acfd726fe263834ad1fe4b354753cd93cf85a511201dd569064eded36475469e", + "https://deno.land/x/transform@v0.4.0/transformers/zlib/gzip.ts": "8008eb620908025c9768bf2fe43c542b122224deb13ce37034237ed44fa97827", + "https://deno.land/x/transform@v0.4.0/transformers/zlib/mod.ts": "ed83911768d491342fb3e09a9b23217c9345ce0b721c69346ed53416d203e195", + "https://deno.land/x/transform@v0.4.0/transformers/zlib/pako.js": "f7077bb75f3f8a479fc945cffa93faa113be1d6c1d1c6d8566d35a6406ad40b8", + "https://raw.githubusercontent.com/marcosc90/encoding-wasm/v0.1.0/dist/encoding_wasm.js": "d7bb9e589b3505f75d220eadb127d98a3b0ef19a884bafc8f94db7a90a24eacf", + "https://raw.githubusercontent.com/marcosc90/encoding-wasm/v0.1.0/dist/encoding_wasm_bytes.js": "c826533d17099db2112c9ddb3cbcd013f7afef5273f4666ac7e0196cc6424659", + "https://raw.githubusercontent.com/marcosc90/encoding-wasm/v0.1.0/mod.ts": "0ebef5c2b134fa6b80ebd52242b23cd19b0dd8d661d07d3fb6cef7b971da99f7" + } +} diff --git a/mod.ts b/mod.ts index ce54022..79d7260 100644 --- a/mod.ts +++ b/mod.ts @@ -10,7 +10,7 @@ import { } from "./deps.ts"; export const CACHE_DIR = join((await cacheDir()) ?? ".cache", "__pick_cache0"); -Deno.mkdir(CACHE_DIR).catch((_) => {}); +await Deno.mkdir(CACHE_DIR).catch((_) => {}); type TarEntry = TarMeta & Deno.Reader; interface GithubPickOptions { @@ -19,11 +19,19 @@ interface GithubPickOptions { pick: RegExp[]; } +/** + * Pick files from a github repo. + * @param {GithubPickOptions} opts - { repo: "denoland/deno", version: "v1.0.0", pick: [/\.ts$/] } + */ export async function* githubPick({ repo, version, pick }: GithubPickOptions) { + await Deno.mkdir(join(CACHE_DIR, repo.split("/").shift() ?? "")).catch( + (_) => {}, + ); const _cached = await getFetchCache(`${repo}@${version}`); if (_cached) { const reader = await Deno.open(_cached); yield* tarGzPickFiles(reader, pick); + reader.close(); } else { yield* githubPickFiles({ repo, version, pick }); } @@ -41,9 +49,11 @@ export async function* githubPickFiles( const targz = await fetch( `https://github.com/${repo}/archive/refs/tags/${version}.tar.gz`, ); - await putFetchCache(targz.body, `${repo}@${version}`); - const reader = readerFromStreamReader(targz.body!.getReader()); - yield* tarGzPickFiles(reader, pick); + const name = `${repo}@${version}`; + await putFetchCache(targz.body, name); + const file = await Deno.open(join(CACHE_DIR, name)); + yield* tarGzPickFiles(file, pick); + file.close(); } /** @@ -77,7 +87,10 @@ export async function* tarGzPickFiles( */ export async function putFetchCache(body: Response["body"], name: string) { const reader = readerFromStreamReader(body!.getReader()); - const writer = await Deno.open(join(CACHE_DIR, name), { write: true }); + const writer = await Deno.open(join(CACHE_DIR, name), { + write: true, + create: true, + }); await copy(reader, writer); writer.close(); } diff --git a/readme.md b/readme.md index 923f653..54dd389 100644 --- a/readme.md +++ b/readme.md @@ -1,3 +1,3 @@ # Pick [WIP] -Pick files from remote sources \ No newline at end of file +Pick files from remote sources diff --git a/tests/mod_test.ts b/tests/mod_test.ts index a191036..a3cf667 100644 --- a/tests/mod_test.ts +++ b/tests/mod_test.ts @@ -1 +1,28 @@ +import { assert } from "https://deno.land/std@0.144.0/_util/assert.ts"; +import {join} from '../deps.ts'; +import * as mod from "../mod.ts"; const { test } = Deno; + +test("mod", async (t) => { + const repo = "saadeghi/daisyui"; + const version = "v2.47.0"; + await t.step("githubPick", async () => { + const pick = [ + /^.*\/base\/.*\.css$/, + /src\/index\.js/, + ]; + const files = []; + for await (const file of mod.githubPick({ repo, version, pick })) { + files.push(file); + } + assert(files.length > 0); + assert(files.filter(f => f.fileName.endsWith(".js")).length == 1); + assert(files.filter(f => f.fileName.endsWith(".css")).length == 2); + }); + await t.step("is cached", async () => { + const name = `${repo}@${version}`; + const cached = await mod.getFetchCache(name); + assert(cached?.endsWith(version)); + assert(cached == join(mod.CACHE_DIR, name)); + }); +});