opensource/coder
3
0
Fork 0
mirror of https://github.com/coder/coder.git synced 2026-09-21 20:51:01 +08:00
Code Issues Packages Projects Releases Wiki Activity

refactor(site): Remove untar dep and support nested folders on template version page (#6244)

Browse Source
This commit is contained in:
Bruno Quaresma
2023-02-16 18:38:14 +00:00
committed by GitHub
parent 8360357834
commit 2c309194e9
11 changed files with 95 additions and 132 deletions
Download Patch File Download Diff File Expand all files Collapse all files
site/js-untar.d.ts
Vendored
-23
View File
@@ -1,23 +0,0 @@
declare module "js-untar" {
export interface File {
name: string
mode: string
blob: Blob
gid: number
uid: number
mtime: number
gname: string
uname: string
type: "0" | "1" | "2" | "3" | "4" | "5" //https://en.wikipedia.org/wiki/Tar_(computing) on Type flag field
}
const Untar: (buffer: ArrayBuffer) => {
then: (
resolve?: (files: File[]) => void,
reject?: () => Promise<void>,
progress?: (file: File) => Promise<void>,
) => Promise<void>
}
export default Untar
}
site/package.json
-2
View File
@@ -59,7 +59,6 @@
"history": "5.3.0",
"i18next": "21.9.1",
"jest-location-mock": "1.0.9",
"js-untar": "2.0.0",
"just-debounce-it": "3.1.1",
"lodash": "4.17.21",
"playwright": "^1.29.2",
@@ -103,7 +102,6 @@
"@types/react-helmet": "6.1.5",
"@types/react-syntax-highlighter": "15.5.5",
"@types/semver": "7.3.12",
"@types/tar-js": "^0.3.2",
"@types/ua-parser-js": "0.7.36",
"@types/uuid": "8.3.4",
"@typescript-eslint/eslint-plugin": "5.50.0",
site/site.go
-2
View File
@@ -326,8 +326,6 @@ func cspHeaders(next http.Handler) http.Handler {
// Report all violations back to the server to log
CSPDirectiveReportURI: {"/api/v2/csp/reports"},
CSPFrameAncestors: {"'none'"},
// worker for loading the .tar files on FE using js-untar
CSPDirectiveWorkerSrc: {"'self' blob:"},
// Only scripts can manipulate the dom. This prevents someone from
// naming themselves something like '<svg onload="alert(/cross-site-scripting/)" />'.
site/src/pages/TemplateVersionPage/TemplateVersionEditorPage/TemplateVersionEditorPage.tsx
+1 -1
View File
@@ -27,7 +27,7 @@ export const TemplateVersionEditorPage: FC = () => {
},
{
onSuccess(data) {
sendEvent({ type: "INITIALIZE", untarFiles: data.untarFiles })
sendEvent({ type: "INITIALIZE", tarReader: data.tarReader })
},
},
)
site/src/pages/TemplateVersionPage/TemplateVersionEditorPage/data.ts
+5 -7
View File
@@ -1,7 +1,7 @@
import { useQuery, UseQueryOptions } from "@tanstack/react-query"
import { getFile, getTemplateByName, getTemplateVersionByName } from "api/api"
import { TarReader } from "util/tar"
import { createTemplateVersionFileTree } from "util/templateVersion"
import untar, { File as UntarFile } from "js-untar"
const getTemplateVersionData = async (
orgId: string,
@@ -13,17 +13,15 @@ const getTemplateVersionData = async (
getTemplateVersionByName(orgId, templateName, versionName),
])
const tarFile = await getFile(version.job.file_id)
let untarFiles: UntarFile[] = []
await untar(tarFile).then((files) => {
untarFiles = files
})
const fileTree = await createTemplateVersionFileTree(untarFiles)
const tarReader = new TarReader()
await tarReader.readFile(tarFile)
const fileTree = await createTemplateVersionFileTree(tarReader)
return {
template,
version,
fileTree,
untarFiles,
tarReader,
}
}
site/src/util/tar.test.ts
+10 -2
View File
@@ -9,7 +9,12 @@ test("tar", async () => {
writer.addFile("b.txt", new Blob(["world"]), { mtime })
writer.addFile("c.txt", "", { mtime })
writer.addFolder("etc", { mtime })
writer.addFile("etc/d.txt", "", { mtime })
writer.addFile("etc/d.txt", "Some text content", {
mtime,
user: "coder",
group: "codergroup",
mode: parseInt("777", 8),
})
const blob = await writer.write()
// Read
@@ -32,8 +37,11 @@ test("tar", async () => {
})
verifyFile(fileInfos[4], reader.getTextFile(fileInfos[4].name) as string, {
name: "etc/d.txt",
content: "",
content: "Some text content",
})
expect(fileInfos[4].group).toEqual("codergroup")
expect(fileInfos[4].user).toEqual("coder")
expect(fileInfos[4].mode).toEqual(parseInt("777", 8))
})
function verifyFile(
site/src/util/tar.ts
+35 -7
View File
@@ -13,6 +13,10 @@ export interface ITarFileInfo {
name: string
type: TarFileType
size: number
mode: number
mtime: number
user: string
group: string
headerOffset: number
}
@@ -34,7 +38,7 @@ export interface ITarWriteOptions {
}
export class TarReader {
private fileInfo: ITarFileInfo[] = []
public fileInfo: ITarFileInfo[] = []
private _buffer: ArrayBuffer | null = null
constructor() {
@@ -64,22 +68,28 @@ export class TarReader {
private readFileInfo() {
this.fileInfo = []
let offset = 0
let fileSize = 0
let fileName = ""
let fileType: TarFileType
while (offset < this.buffer.byteLength - 512) {
fileName = this.readFileName(offset)
const fileName = this.readFileName(offset)
if (!fileName) {
break
}
fileType = this.readFileType(offset)
fileSize = this.readFileSize(offset)
const fileType = this.readFileType(offset)
const fileSize = this.readFileSize(offset)
const fileMode = this.readFileMode(offset)
const fileMtime = this.readFileMtime(offset)
const fileUser = this.readFileUser(offset)
const fileGroup = this.readFileGroup(offset)
this.fileInfo.push({
name: fileName,
type: fileType,
size: fileSize,
headerOffset: offset,
mode: fileMode,
mtime: fileMtime,
user: fileUser,
group: fileGroup,
})
offset += 512 + 512 * Math.floor((fileSize + 511) / 512)
@@ -100,6 +110,24 @@ export class TarReader {
return this.readString(offset, 100)
}
private readFileMode(offset: number) {
const mode = this.readString(offset + 100, 8)
return parseInt(mode, 8)
}
private readFileMtime(offset: number) {
const mtime = this.readString(offset + 136, 12)
return parseInt(mtime, 8)
}
private readFileUser(offset: number) {
return this.readString(offset + 265, 32)
}
private readFileGroup(offset: number) {
return this.readString(offset + 297, 32)
}
private readFileType(offset: number) {
const typeView = new Uint8Array(this.buffer, offset + 156, 1)
const typeStr = String.fromCharCode(typeView[0])
site/src/util/templateVersion.ts
+15 -37
View File
@@ -1,7 +1,7 @@
import * as API from "api/api"
import { TemplateVersion } from "api/typesGenerated"
import untar, { File as UntarFile } from "js-untar"
import { FileTree, setFile } from "./filetree"
import { TarReader } from "./tar"
/**
* Content by filename
@@ -10,32 +10,16 @@ export type TemplateVersionFiles = Record<string, string>
export const getTemplateVersionFiles = async (
version: TemplateVersion,
allowedExtensions: string[],
allowedFiles: string[],
): Promise<TemplateVersionFiles> => {
const files: TemplateVersionFiles = {}
const tarFile = await API.getFile(version.job.file_id)
const blobs: Record<string, Blob> = {}
await untar(tarFile).then(undefined, undefined, async (file) => {
const paths = file.name.split("/")
const filename = paths[paths.length - 1]
const [_, extension] = filename.split(".")
if (
allowedExtensions.includes(extension) ||
allowedFiles.includes(filename)
) {
blobs[filename] = file.blob
const tarReader = new TarReader()
await tarReader.readFile(tarFile)
for (const file of tarReader.fileInfo) {
if (isAllowedFile(file.name)) {
files[file.name] = tarReader.getTextFile(file.name) as string
}
})
await Promise.all(
Object.entries(blobs).map(async ([filename, blob]) => {
files[filename] = await blob.text()
}),
)
}
return files
}
@@ -46,23 +30,17 @@ export const isAllowedFile = (name: string) => {
}
export const createTemplateVersionFileTree = async (
untarFiles: UntarFile[],
tarReader: TarReader,
): Promise<FileTree> => {
let fileTree: FileTree = {}
const blobs: Record<string, Blob> = {}
for (const untarFile of untarFiles) {
if (isAllowedFile(untarFile.name)) {
blobs[untarFile.name] = untarFile.blob
for (const file of tarReader.fileInfo) {
if (isAllowedFile(file.name)) {
fileTree = setFile(
file.name,
tarReader.getTextFile(file.name) as string,
fileTree,
)
}
}
await Promise.all(
Object.entries(blobs).map(async ([fullPath, blob]) => {
const content = await blob.text()
fileTree = setFile(fullPath, content, fileTree)
}),
)
return fileTree
}
site/src/xServices/templateVersion/templateVersionXService.ts
+2 -16
View File
@@ -162,23 +162,9 @@ export const templateVersionMachine = createMachine(
}
const loadFilesPromises: ReturnType<typeof getTemplateVersionFiles>[] =
[]
const allowedExtensions = ["tf", "md"]
const allowedFiles = ["Dockerfile"]
loadFilesPromises.push(
getTemplateVersionFiles(
currentVersion,
allowedExtensions,
allowedFiles,
),
)
loadFilesPromises.push(getTemplateVersionFiles(currentVersion))
if (previousVersion) {
loadFilesPromises.push(
getTemplateVersionFiles(
previousVersion,
allowedExtensions,
allowedFiles,
),
)
loadFilesPromises.push(getTemplateVersionFiles(previousVersion))
}
const [currentFiles, previousFiles] = await Promise.all(
loadFilesPromises,
site/src/xServices/templateVersionEditor/templateVersionEditorXService.ts
+27 -25
View File
@@ -7,10 +7,9 @@ import {
} from "api/typesGenerated"
import { assign, createMachine } from "xstate"
import * as API from "api/api"
import { File as UntarFile } from "js-untar"
import { FileTree, traverse } from "util/filetree"
import { isAllowedFile } from "util/templateVersion"
import { TarWriter } from "util/tar"
import { TarReader, TarWriter } from "util/tar"
export interface CreateVersionData {
file: File
@@ -24,7 +23,7 @@ export interface TemplateVersionEditorMachineContext {
version?: TemplateVersion
resources?: WorkspaceResource[]
buildLogs?: ProvisionerJobLog[]
untarFiles?: UntarFile[]
tarReader?: TarReader
}
export const templateVersionEditorMachine = createMachine(
@@ -34,7 +33,7 @@ export const templateVersionEditorMachine = createMachine(
schema: {
context: {} as TemplateVersionEditorMachineContext,
events: {} as
| { type: "INITIALIZE"; untarFiles: UntarFile[] }
| { type: "INITIALIZE"; tarReader: TarReader }
| {
type: "CREATE_VERSION"
fileTree: FileTree
@@ -70,7 +69,7 @@ export const templateVersionEditorMachine = createMachine(
initializing: {
on: {
INITIALIZE: {
actions: ["assignUntarFiles"],
actions: ["assignTarReader"],
target: "idle",
},
},
@@ -213,38 +212,41 @@ export const templateVersionEditorMachine = createMachine(
}
},
}),
assignUntarFiles: assign({
untarFiles: (_, { untarFiles }) => untarFiles,
assignTarReader: assign({
tarReader: (_, { tarReader }) => tarReader,
}),
},
services: {
uploadTar: async ({ fileTree, untarFiles }) => {
uploadTar: async ({ fileTree, tarReader }) => {
if (!fileTree) {
throw new Error("file tree must to be set")
}
if (!untarFiles) {
throw new Error("untar files must to be set")
if (!tarReader) {
throw new Error("tar reader must to be set")
}
const tar = new TarWriter()
// Add previous non editable files
for (const untarFile of untarFiles) {
if (!isAllowedFile(untarFile.name)) {
if (untarFile.type === "5") {
tar.addFolder(untarFile.name, {
mode: parseInt(untarFile.mode, 8) & 0xfff, // https://github.com/beatgammit/tar-js/blob/master/lib/tar.js#L42
mtime: untarFile.mtime,
user: untarFile.uname,
group: untarFile.gname,
for (const file of tarReader.fileInfo) {
if (!isAllowedFile(file.name)) {
if (file.type === "5") {
tar.addFolder(file.name, {
mode: file.mode, // https://github.com/beatgammit/tar-js/blob/master/lib/tar.js#L42
mtime: file.mtime,
user: file.user,
group: file.group,
})
} else {
const buffer = await untarFile.blob.arrayBuffer()
tar.addFile(untarFile.name, new Uint8Array(buffer), {
mode: parseInt(untarFile.mode, 8) & 0xfff, // https://github.com/beatgammit/tar-js/blob/master/lib/tar.js#L42
mtime: untarFile.mtime,
user: untarFile.uname,
group: untarFile.gname,
})
tar.addFile(
file.name,
tarReader.getTextFile(file.name) as string,
{
mode: file.mode, // https://github.com/beatgammit/tar-js/blob/master/lib/tar.js#L42
mtime: file.mtime,
user: file.user,
group: file.group,
},
)
}
}
}
site/yarn.lock
-10
View File
@@ -3455,11 +3455,6 @@
resolved "https://registry.yarnpkg.com/@types/tapable/-/tapable-1.0.8.tgz#b94a4391c85666c7b73299fd3ad79d4faa435310"
integrity sha512-ipixuVrh2OdNmauvtT51o3d8z12p6LtFW9in7U79der/kwejjdNchQC5UMn5u/KxNoM7VHHOs/l8KS8uHxhODQ==
"@types/tar-js@^0.3.2":
version "0.3.2"
resolved "https://registry.yarnpkg.com/@types/tar-js/-/tar-js-0.3.2.tgz#23d3c7c8ce22bec6ed8355c2169dd3fd6ebe2583"
integrity sha512-0ySOBNP+/Ey67EZ0QaCOgkt6zSAeayTwsoln02ztlyB5lF4NQD0sl5C7E5eKS+QUb7xgTEPdPo9LUjYOHUJVqQ==
"@types/testing-library__jest-dom@^5.9.1":
version "5.14.5"
resolved "https://registry.yarnpkg.com/@types/testing-library__jest-dom/-/testing-library__jest-dom-5.14.5.tgz#d113709c90b3c75fdb127ec338dad7d5f86c974f"
@@ -9492,11 +9487,6 @@ js-string-escape@^1.0.1:
resolved "https://registry.yarnpkg.com/js-tokens/-/js-tokens-4.0.0.tgz#19203fb59991df98e3a287050d4647cdeaf32499"
integrity sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==
js-untar@2.0.0:
version "2.0.0"
resolved "https://registry.yarnpkg.com/js-untar/-/js-untar-2.0.0.tgz#b452d28dedd3b0be92c2ac9a7d70f612a93c7453"
integrity sha512-7CsDLrYQMbLxDt2zl9uKaPZSdmJMvGGQ7wo9hoB3J+z/VcO2w63bXFgHVnjF1+S9wD3zAu8FBVj7EYWjTQ3Z7g==
js-yaml@^3.10.0, js-yaml@^3.13.1:
version "3.14.1"
resolved "https://registry.yarnpkg.com/js-yaml/-/js-yaml-3.14.1.tgz#dae812fdb3825fa306609a8717383c50c36a0537"