x/pickit
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

pick files from tar.gz and github

Browse source
This commit is contained in:
Anton Nesterov 2023-02-10 19:30:37 +03:00
parent 3d03526471
commit a4f67c1495
5 changed files with 101 additions and 6 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
deno.json Normal file
View file

@ -0,0 +1,8 @@
{
"tasks": {
"test": "deno test -A --no-check tests/"
},
"compilerOptions": {
"strict": true
}
}

47
deno.lock Normal file
View file

@ -0,0 +1,47 @@
{
"version": "2",
"remote": {
"https://deno.land/std@0.144.0/_util/assert.ts": "e94f2eb37cebd7f199952e242c77654e43333c1ac4c5c700e929ea3aa5489f74",
"https://deno.land/std@0.144.0/_util/os.ts": "3b4c6e27febd119d36a416d7a97bd3b0251b77c88942c8f16ee5953ea13e2e49",
"https://deno.land/std@0.144.0/archive/tar.ts": "ea420312bb9105d5c960133563d0eacf0f2f86e097e847cd84c2172c687a1125",
"https://deno.land/std@0.144.0/async/deferred.ts": "bc18e28108252c9f67dfca2bbc4587c3cbf3aeb6e155f8c864ca8ecff992b98a",
"https://deno.land/std@0.144.0/bytes/bytes_list.ts": "aba5e2369e77d426b10af1de0dcc4531acecec27f9b9056f4f7bfbf8ac147ab4",
"https://deno.land/std@0.144.0/bytes/equals.ts": "3c3558c3ae85526f84510aa2b48ab2ad7bdd899e2e0f5b7a8ffc85acb3a6043a",
"https://deno.land/std@0.144.0/bytes/mod.ts": "763f97d33051cc3f28af1a688dfe2830841192a9fea0cbaa55f927b49d49d0bf",
"https://deno.land/std@0.144.0/io/buffer.ts": "bd0c4bf53db4b4be916ca5963e454bddfd3fcd45039041ea161dbf826817822b",
"https://deno.land/std@0.144.0/io/readers.ts": "45847ad404afd2f605eae1cff193f223462bc55eeb9ae313c2f3db28aada0fd6",
"https://deno.land/std@0.144.0/io/types.d.ts": "0cae3a62da7a37043661746c65c021058bae020b54e50c0e774916e5d4baee43",
"https://deno.land/std@0.144.0/path/_constants.ts": "df1db3ffa6dd6d1252cc9617e5d72165cd2483df90e93833e13580687b6083c3",
"https://deno.land/std@0.144.0/path/_interface.ts": "ee3b431a336b80cf445441109d089b70d87d5e248f4f90ff906820889ecf8d09",
"https://deno.land/std@0.144.0/path/_util.ts": "c1e9686d0164e29f7d880b2158971d805b6e0efc3110d0b3e24e4b8af2190d2b",
"https://deno.land/std@0.144.0/path/common.ts": "bee563630abd2d97f99d83c96c2fa0cca7cee103e8cb4e7699ec4d5db7bd2633",
"https://deno.land/std@0.144.0/path/glob.ts": "cb5255638de1048973c3e69e420c77dc04f75755524cb3b2e160fe9277d939ee",
"https://deno.land/std@0.144.0/path/mod.ts": "4945b430b759b0b3d98f2a278542cbcf95e0ad2bd8eaaed3c67322b306b2b346",
"https://deno.land/std@0.144.0/path/posix.ts": "293cdaec3ecccec0a9cc2b534302dfe308adb6f10861fa183275d6695faace44",
"https://deno.land/std@0.144.0/path/separator.ts": "fe1816cb765a8068afb3e8f13ad272351c85cbc739af56dacfc7d93d710fe0f9",
"https://deno.land/std@0.144.0/path/win32.ts": "31811536855e19ba37a999cd8d1b62078235548d67902ece4aa6b814596dd757",
"https://deno.land/std@0.144.0/streams/buffer.ts": "ee47194022d47fa23d4749b8afbadc83c237c4595467a9bddb322af0dd205815",
"https://deno.land/std@0.144.0/streams/conversion.ts": "fc3db02026183da795fa32ac7549868e9f19c75ba029d4b4c3739af62b48517a",
"https://deno.land/std@0.144.0/streams/delimiter.ts": "81574a35e433d3b392887118d02fce12fec611fe3ff6b8f3d3758d141b7201e9",
"https://deno.land/std@0.144.0/streams/merge.ts": "88ed3dfa030ae076802688e4cadd762a251a41d81ed1776dfd9a2a9a0e970195",
"https://deno.land/std@0.144.0/streams/mod.ts": "d7d8fa97f79bca052eba8b6df9c4602403d144cf97ec9f6d0896c787dae0a9de",
"https://deno.land/std@0.57.0/async/deferred.ts": "ac95025f46580cf5197928ba90995d87f26e202c19ad961bc4e3177310894cdc",
"https://deno.land/x/cache_dir@0.2.0/mod.ts": "89ac1bc4d8f1c52fde192e0470d0597fa156ecf2b53d7fe162a11fd365c79920",
"https://deno.land/x/transform@v0.2.0/transform.ts": "2d3690dae85be1ec7a6e09025ef651e0b683910f7470b474adf59406df917da6",
"https://deno.land/x/transform@v0.4.0/mod.ts": "6e039b9e4bdbc0cb57deb48c955f5e26fe7251b7619ea7d79105f95487683838",
"https://deno.land/x/transform@v0.4.0/transform.ts": "2d3690dae85be1ec7a6e09025ef651e0b683910f7470b474adf59406df917da6",
"https://deno.land/x/transform@v0.4.0/transformers/base64/decoder.ts": "7d533dd86cfbb097a89eb83d6e9539a3330b631f11975d8774716c234178793a",
"https://deno.land/x/transform@v0.4.0/transformers/base64/deps.ts": "f56c87286aee9b254ed008219dacabd701a559d5c0f8eb7ac7bafcf3ffe480d3",
"https://deno.land/x/transform@v0.4.0/transformers/base64/encoder.ts": "9077a79488ea9bdf34203c5056ea6c225eda38503282fed26746b26d5d2779e9",
"https://deno.land/x/transform@v0.4.0/transformers/base64/mod.ts": "6b555deb6b31319f043106c6816aa03d80b3965f9badad1befb10ee303b764b4",
"https://deno.land/x/transform@v0.4.0/transformers/base64/util.ts": "cd9f13c3a03c43c34e605c65dd1e0655711ca9fbbf6c4663ecf0213009bb96c4",
"https://deno.land/x/transform@v0.4.0/transformers/errors.ts": "958ddc017ed11067ffff470acdae4c6107c2de3a20fa2df28c2ee3586596d647",
"https://deno.land/x/transform@v0.4.0/transformers/mod.ts": "acfd726fe263834ad1fe4b354753cd93cf85a511201dd569064eded36475469e",
"https://deno.land/x/transform@v0.4.0/transformers/zlib/gzip.ts": "8008eb620908025c9768bf2fe43c542b122224deb13ce37034237ed44fa97827",
"https://deno.land/x/transform@v0.4.0/transformers/zlib/mod.ts": "ed83911768d491342fb3e09a9b23217c9345ce0b721c69346ed53416d203e195",
"https://deno.land/x/transform@v0.4.0/transformers/zlib/pako.js": "f7077bb75f3f8a479fc945cffa93faa113be1d6c1d1c6d8566d35a6406ad40b8",
"https://raw.githubusercontent.com/marcosc90/encoding-wasm/v0.1.0/dist/encoding_wasm.js": "d7bb9e589b3505f75d220eadb127d98a3b0ef19a884bafc8f94db7a90a24eacf",
"https://raw.githubusercontent.com/marcosc90/encoding-wasm/v0.1.0/dist/encoding_wasm_bytes.js": "c826533d17099db2112c9ddb3cbcd013f7afef5273f4666ac7e0196cc6424659",
"https://raw.githubusercontent.com/marcosc90/encoding-wasm/v0.1.0/mod.ts": "0ebef5c2b134fa6b80ebd52242b23cd19b0dd8d661d07d3fb6cef7b971da99f7"
}
}

23
mod.ts
View file

@ -10,7 +10,7 @@ import {
} from "./deps.ts";
export const CACHE_DIR = join((await cacheDir()) ?? ".cache", "__pick_cache0");
Deno.mkdir(CACHE_DIR).catch((_) => {});
await Deno.mkdir(CACHE_DIR).catch((_) => {});
type TarEntry = TarMeta & Deno.Reader;
interface GithubPickOptions {
@ -19,11 +19,19 @@ interface GithubPickOptions {
pick: RegExp[];
}
/**
* Pick files from a github repo.
* @param {GithubPickOptions} opts - { repo: "denoland/deno", version: "v1.0.0", pick: [/\.ts$/] }
*/
export async function* githubPick({ repo, version, pick }: GithubPickOptions) {
await Deno.mkdir(join(CACHE_DIR, repo.split("/").shift() ?? "")).catch(
(_) => {},
);
const _cached = await getFetchCache(`${repo}@${version}`);
if (_cached) {
const reader = await Deno.open(_cached);
yield* tarGzPickFiles(reader, pick);
reader.close();
} else {
yield* githubPickFiles({ repo, version, pick });
}
@ -41,9 +49,11 @@ export async function* githubPickFiles(
const targz = await fetch(
`https://github.com/${repo}/archive/refs/tags/${version}.tar.gz`,
);
await putFetchCache(targz.body, `${repo}@${version}`);
const reader = readerFromStreamReader(targz.body!.getReader());
yield* tarGzPickFiles(reader, pick);
const name = `${repo}@${version}`;
await putFetchCache(targz.body, name);
const file = await Deno.open(join(CACHE_DIR, name));
yield* tarGzPickFiles(file, pick);
file.close();
}
/**
@ -77,7 +87,10 @@ export async function* tarGzPickFiles(
*/
export async function putFetchCache(body: Response["body"], name: string) {
const reader = readerFromStreamReader(body!.getReader());
const writer = await Deno.open(join(CACHE_DIR, name), { write: true });
const writer = await Deno.open(join(CACHE_DIR, name), {
write: true,
create: true,
});
await copy(reader, writer);
writer.close();
}

2
readme.md
View file

@ -1,3 +1,3 @@
# Pick [WIP]
Pick files from remote sources
Pick files from remote sources

27
tests/mod_test.ts
View file

@ -1 +1,28 @@
import { assert } from "https://deno.land/std@0.144.0/_util/assert.ts";
import {join} from '../deps.ts';
import * as mod from "../mod.ts";
const { test } = Deno;
test("mod", async (t) => {
const repo = "saadeghi/daisyui";
const version = "v2.47.0";
await t.step("githubPick", async () => {
const pick = [
/^.*\/base\/.*\.css$/,
/src\/index\.js/,
];
const files = [];
for await (const file of mod.githubPick({ repo, version, pick })) {
files.push(file);
}
assert(files.length > 0);
assert(files.filter(f => f.fileName.endsWith(".js")).length == 1);
assert(files.filter(f => f.fileName.endsWith(".css")).length == 2);
});
await t.step("is cached", async () => {
const name = `${repo}@${version}`;
const cached = await mod.getFetchCache(name);
assert(cached?.endsWith(version));
assert(cached == join(mod.CACHE_DIR, name));
});
});