Skip to content

Commit

Permalink
Merge pull request #1015 from microsoft/benibenj/scrawny-flyingfish
Browse files Browse the repository at this point in the history 
Support "ls --tree"
  • Loading branch information
@benibenj
benibenj authored Jul 16, 2024
2 parents 7dc3477 + 4e81044 commit 826b4e4
Show file tree
Hide file tree
Showing 3 changed files with 187 additions and 73 deletions.
5 changes: 3 additions & 2 deletions src/main.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,7 @@ module.exports = function (argv: string[]): void {
program
.command('ls')
.description('Lists all the files that will be published/packaged')
.option('--tree', 'Prints the files in a tree format', false)
.option('--yarn', 'Use yarn instead of npm (default inferred from presence of yarn.lock or .yarnrc)')
.option('--no-yarn', 'Use npm instead of yarn (default inferred from absence of yarn.lock or .yarnrc)')
.option<string[]>(
Expand All @@ -73,8 +74,8 @@ module.exports = function (argv: string[]): void {
// default must remain undefined for dependencies or we will fail to load defaults from package.json
.option('--dependencies', 'Enable dependency detection via npm or yarn', undefined)
.option('--no-dependencies', 'Disable dependency detection via npm or yarn', undefined)
.action(({ yarn, packagedDependencies, ignoreFile, dependencies }) =>
main(ls({ useYarn: yarn, packagedDependencies, ignoreFile, dependencies }))
.action(({ tree, yarn, packagedDependencies, ignoreFile, dependencies }) =>
main(ls({ tree, useYarn: yarn, packagedDependencies, ignoreFile, dependencies }))
);

program
Expand Down
73 changes: 45 additions & 28 deletions src/package.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1828,7 +1828,7 @@ export async function pack(options: IPackageOptions = {}): Promise<IPackageResul
const manifest = await readManifest(cwd);
const files = await collect(manifest, options);

printPackagedFiles(files, cwd, manifest, options);
await printPackagedFiles(files, cwd, manifest, options);

if (options.version && !(options.updatePackageJson ?? true)) {
manifest.version = options.version;
Expand Down Expand Up @@ -1885,23 +1885,13 @@ export async function packageCommand(options: IPackageOptions = {}): Promise<any
}

const stats = await fs.promises.stat(packagePath);

let size = 0;
let unit = '';

if (stats.size > 1048576) {
size = Math.round(stats.size / 10485.76) / 100;
unit = 'MB';
} else {
size = Math.round(stats.size / 10.24) / 100;
unit = 'KB';
}

util.log.done(`Packaged: ${packagePath} (${files.length} files, ${size}${unit})`);
const packageSize = util.bytesToString(stats.size);
util.log.done(`Packaged: ${packagePath} ` + chalk.bold(`(${files.length} files, ${packageSize})`));
}

export interface IListFilesOptions {
readonly cwd?: string;
readonly manifest?: Manifest;
readonly useYarn?: boolean;
readonly packagedDependencies?: string[];
readonly ignoreFile?: string;
Expand All @@ -1914,7 +1904,7 @@ export interface IListFilesOptions {
*/
export async function listFiles(options: IListFilesOptions = {}): Promise<string[]> {
const cwd = options.cwd ?? process.cwd();
const manifest = await readManifest(cwd);
const manifest = options.manifest ?? await readManifest(cwd);

if (options.prepublish) {
await prepublish(cwd, manifest, options.useYarn);
Expand All @@ -1923,29 +1913,46 @@ export async function listFiles(options: IListFilesOptions = {}): Promise<string
return await collectFiles(cwd, getDependenciesOption(options), options.packagedDependencies, options.ignoreFile, manifest.files);
}

interface ILSOptions {
readonly tree?: boolean;
readonly useYarn?: boolean;
readonly packagedDependencies?: string[];
readonly ignoreFile?: string;
readonly dependencies?: boolean;
}

/**
* Lists the files included in the extension's package. Runs prepublish.
* Lists the files included in the extension's package.
*/
export async function ls(options: IListFilesOptions = {}): Promise<void> {
const files = await listFiles({ ...options, prepublish: true });
export async function ls(options: ILSOptions = {}): Promise<void> {
const cwd = process.cwd();
const manifest = await readManifest(cwd);

for (const file of files) {
console.log(`${file}`);
const files = await listFiles({ ...options, cwd, manifest });

if (options.tree) {
const printableFileStructure = await util.generateFileStructureTree(
getDefaultPackageName(manifest, options),
files.map(f => ({ origin: f, tree: f }))
);
console.log(printableFileStructure.join('\n'));
} else {
console.log(files.join('\n'));
}
}

/**
* Prints the packaged files of an extension.
*/
export function printPackagedFiles(files: IFile[], cwd: string, manifest: Manifest, options: IPackageOptions): void {
export async function printPackagedFiles(files: IFile[], cwd: string, manifest: Manifest, options: IPackageOptions): Promise<void> {
// Warn if the extension contains a lot of files
const jsFiles = files.filter(f => /\.js$/i.test(f.path));
if (files.length > 5000 || jsFiles.length > 100) {
let message = '\n';
let message = '';
message += `This extension consists of ${chalk.bold(String(files.length))} files, out of which ${chalk.bold(String(jsFiles.length))} are JavaScript files. `;
message += `For performance reasons, you should bundle your extension: ${chalk.underline('https://aka.ms/vscode-bundle-extension')}. `;
message += `You should also exclude unnecessary files by adding them to your .vscodeignore: ${chalk.underline('https://aka.ms/vscode-vscodeignore')}.\n`;
console.log(message);
util.log.warn(message);
}

// Warn if the extension does not have a .vscodeignore file or a files property in package.json
Expand All @@ -1954,23 +1961,33 @@ export function printPackagedFiles(files: IFile[], cwd: string, manifest: Manife
if (!hasDeaultIgnore) {
let message = '';
message += `Neither a ${chalk.bold('.vscodeignore')} file nor a ${chalk.bold('"files"')} property in package.json was found. `;
message += `To ensure only necessary files are included in your extension package, `;
message += `add a .vscodeignore file or specify the "files" property in package.json. More info: ${chalk.underline('https://aka.ms/vscode-vscodeignore')}`;
message += `To ensure only necessary files are included in your extension, `;
message += `add a .vscodeignore file or specify the "files" property in package.json. More info: ${chalk.underline('https://aka.ms/vscode-vscodeignore')}\n`;
util.log.warn(message);
}
}

// Print the files included in the package
const printableFileStructure = util.generateFileStructureTree(getDefaultPackageName(manifest, options), files.map(f => f.path), 35);
const printableFileStructure = await util.generateFileStructureTree(
getDefaultPackageName(manifest, options),
files.map(f => ({
// File path relative to the extension root
origin: f.path.startsWith('extension/') ? f.path.substring(10) : f.path,
// File path in the VSIX
tree: f.path
})),
35 // Print up to 35 files/folders
);

let message = '';
message += chalk.bold.blue(`Files included in the VSIX:\n`);
message += printableFileStructure.join('\n');

// If not all files have been printed, mention how all files can be printed
if (files.length + 1 > printableFileStructure.length) {
// If not all files have been printed, mention how all files can be printed
message += `\n\n=> Run ${chalk.bold('vsce ls')} to see a list of all included files.\n`;
message += `\n\n=> Run ${chalk.bold('vsce ls --tree')} to see all included files.`;
}

message += '\n';
util.log.info(message);
}
182 changes: 139 additions & 43 deletions src/util.ts
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import { promisify } from 'util';
import * as fs from 'fs';
import _read from 'read';
import { WebApi, getBasicHandler } from 'azure-devops-node-api/WebApi';
import { IGalleryApi, GalleryApi } from 'azure-devops-node-api/GalleryApi';
Expand Down Expand Up @@ -184,88 +185,183 @@ export function patchOptionsWithManifest(options: any, manifest: Manifest): void
}
}

export function generateFileStructureTree(rootFolder: string, filePaths: string[], maxPrint: number = Number.MAX_VALUE): string[] {
export function bytesToString(bytes: number): string {
let size = 0;
let unit = '';

if (bytes > 1048576) {
size = Math.round(bytes / 10485.76) / 100;
unit = 'MB';
} else {
size = Math.round(bytes / 10.24) / 100;
unit = 'KB';
}
return `${size} ${unit}`;
}

const FOLDER_SIZE_KEY = "/__FOlDER_SIZE__\\";
const FOLDER_FILES_TOTAL_KEY = "/__FOLDER_CHILDREN__\\";
const FILE_SIZE_WARNING_THRESHOLD = 0.85;
const FILE_SIZE_LARGE_THRESHOLD = 0.2;

export async function generateFileStructureTree(rootFolder: string, filePaths: { origin: string, tree: string }[], printLinesLimit: number = Number.MAX_VALUE): Promise<string[]> {
const folderTree: any = {};
const depthCounts: number[] = [];

// Build a tree structure from the file paths
filePaths.forEach(filePath => {
const parts = filePath.split('/');
// Store the file size in the leaf node and the folder size in the folder node
// Store the number of children in the folder node
for (const filePath of filePaths) {
const parts = filePath.tree.split('/');
let currentLevel = folderTree;

parts.forEach((part, depth) => {
const isFile = depth === parts.length - 1;

// Create the node if it doesn't exist
if (!currentLevel[part]) {
currentLevel[part] = depth === parts.length - 1 ? null : {};
if (isFile) {
// The file size is stored in the leaf node,
currentLevel[part] = 0;
} else {
// The folder size is stored in the folder node
currentLevel[part] = {};
currentLevel[part][FOLDER_SIZE_KEY] = 0;
currentLevel[part][FOLDER_FILES_TOTAL_KEY] = 0;
}

// Count the number of items at each depth
if (depthCounts.length <= depth) {
depthCounts.push(0);
}
depthCounts[depth]++;
}

currentLevel = currentLevel[part];

// Count the total number of children in the nested folders
if (!isFile) {
currentLevel[FOLDER_FILES_TOTAL_KEY]++;
}
});
});
};

// Get max depth
// Get max depth depending on the maximum number of lines allowed to print
let currentDepth = 0;
let countUpToCurrentDepth = depthCounts[0];
let countUpToCurrentDepth = depthCounts[0] + 1 /* root folder */;
for (let i = 1; i < depthCounts.length; i++) {
if (countUpToCurrentDepth + depthCounts[i] > maxPrint) {
if (countUpToCurrentDepth + depthCounts[i] > printLinesLimit) {
break;
}
currentDepth++;
countUpToCurrentDepth += depthCounts[i];
}

const maxDepth = currentDepth;
let message: string[] = [];

// Helper function to print the tree
const printTree = (tree: any, depth: number, prefix: string) => {
// Get all file sizes
const fileSizes: [number, string][] = await Promise.all(filePaths.map(async (filePath) => {
try {
const stats = await fs.promises.stat(filePath.origin);
return [stats.size, filePath.tree];
} catch (error) {
return [0, filePath.origin];
}
}));

// Store all file sizes in the tree
let totalFileSizes = 0;
fileSizes.forEach(([size, filePath]) => {
totalFileSizes += size;

const parts = filePath.split('/');
let currentLevel = folderTree;
parts.forEach(part => {
if (typeof currentLevel[part] === 'number') {
currentLevel[part] = size;
} else if (currentLevel[part]) {
currentLevel[part][FOLDER_SIZE_KEY] += size;
}
currentLevel = currentLevel[part];
});
});

let output: string[] = [];
output.push(chalk.bold(rootFolder));
output.push(...createTreeOutput(folderTree, maxDepth, totalFileSizes));

for (const [size, filePath] of fileSizes) {
if (size > FILE_SIZE_WARNING_THRESHOLD * totalFileSizes) {
output.push(`\nThe file ${filePath} is ${chalk.red('large')} (${bytesToString(size)})`);
break;
}
}

return output;
}

function createTreeOutput(fileSystem: any, maxDepth: number, totalFileSizes: number): string[] {

const getColorFromSize = (size: number) => {
if (size > FILE_SIZE_WARNING_THRESHOLD * totalFileSizes) {
return chalk.red;
} else if (size > FILE_SIZE_LARGE_THRESHOLD * totalFileSizes) {
return chalk.yellow;
} else {
return chalk.grey;
}
};

const createFileOutput = (prefix: string, fileName: string, fileSize: number) => {
let fileSizeColored = '';
if (fileSize > 0) {
const fileSizeString = `[${bytesToString(fileSize)}]`;
fileSizeColored = getColorFromSize(fileSize)(fileSizeString);
}
return `${prefix}${fileName} ${fileSizeColored}`;
}

const createFolderOutput = (prefix: string, filesCount: number, folderSize: number, folderName: string, depth: number) => {
if (depth < maxDepth) {
// Max depth is not reached, print only the folder
// as children will be printed
return prefix + chalk.bold(`${folderName}/`);
}

// Max depth is reached, print the folder name and additional metadata
// as children will not be printed
const folderSizeString = bytesToString(folderSize);
const folder = chalk.bold(`${folderName}/`);
const numFilesString = chalk.green(`(${filesCount} ${filesCount === 1 ? 'file' : 'files'})`);
const folderSizeColored = getColorFromSize(folderSize)(`[${folderSizeString}]`);
return `${prefix}${folder} ${numFilesString} ${folderSizeColored}`;
}

const createTreeLayerOutput = (tree: any, depth: number, prefix: string, path: string) => {
// Print all files before folders
const sortedFolderKeys = Object.keys(tree).filter(key => tree[key] !== null).sort();
const sortedFileKeys = Object.keys(tree).filter(key => tree[key] === null).sort();
const sortedKeys = [...sortedFileKeys, ...sortedFolderKeys];
const sortedFolderKeys = Object.keys(tree).filter(key => typeof tree[key] !== 'number').sort();
const sortedFileKeys = Object.keys(tree).filter(key => typeof tree[key] === 'number').sort();
const sortedKeys = [...sortedFileKeys, ...sortedFolderKeys].filter(key => key !== FOLDER_SIZE_KEY && key !== FOLDER_FILES_TOTAL_KEY);

const output: string[] = [];
for (let i = 0; i < sortedKeys.length; i++) {

const key = sortedKeys[i];
const isLast = i === sortedKeys.length - 1;
const localPrefix = prefix + (isLast ? '└─ ' : '├─ ');
const childPrefix = prefix + (isLast ? ' ' : '│ ');

if (tree[key] === null) {
if (typeof tree[key] === 'number') {
// It's a file
message.push(localPrefix + key);
output.push(createFileOutput(localPrefix, key, tree[key]));
} else {
// It's a folder
output.push(createFolderOutput(localPrefix, tree[key][FOLDER_FILES_TOTAL_KEY], tree[key][FOLDER_SIZE_KEY], key, depth));
if (depth < maxDepth) {
// maxdepth is not reached, print the folder and its children
message.push(localPrefix + chalk.bold(`${key}/`));
printTree(tree[key], depth + 1, childPrefix);
} else {
// max depth is reached, print the folder but not its children
const filesCount = countFiles(tree[key]);
message.push(localPrefix + chalk.bold(`${key}/`) + chalk.green(` (${filesCount} ${filesCount === 1 ? 'file' : 'files'})`));
output.push(...createTreeLayerOutput(tree[key], depth + 1, childPrefix, path + key + '/'));
}
}
}
return output;
};

// Helper function to count the number of files in a tree
const countFiles = (tree: any): number => {
let filesCount = 0;
for (const key in tree) {
if (tree[key] === null) {
filesCount++;
} else {
filesCount += countFiles(tree[key]);
}
}
return filesCount;
};

message.push(chalk.bold(rootFolder));
printTree(folderTree, 0, '');

return message;
}
return createTreeLayerOutput(fileSystem, 0, '', '');
}

0 comments on commit 826b4e4

Please sign in to comment.