add --skip-volume flag to up command (#366)

packages/cli/src/commands/up.ts

@@ -5,7 +5,7 @@ import {
   ProfileStore,
   TunnelOpts,
   addBaseComposeTunnelAgentService,
-  commands, ensureMachine, findComposeTunnelAgentUrl,
+  commands, defaultVolumeSkipList, ensureMachine, findComposeTunnelAgentUrl,
   findEnvId,
   findProjectName, getTunnelNamesToServicePorts, jwkThumbprint, profileStore,
   telemetryEmitter,
@@ -82,6 +82,12 @@ export default class Up extends MachineCreationDriverCommand<typeof Up> {
     ...envIdFlags,
     ...tunnelServerFlags,
     ...buildFlags,
+    'skip-volume': Flags.string({
+      description: 'Additional volume glob patterns to skip copying',
+      multiple: true,
+      singleValue: true,
+      default: [],
+    }),
     'skip-unchanged-files': Flags.boolean({
       description: 'Detect and skip unchanged files when copying (default: true)',
       default: true,
@@ -181,6 +187,7 @@ export default class Up extends MachineCreationDriverCommand<typeof Up> {
       projectName,
       expectedServiceUrls,
       userSpecifiedServices: restArgs,
+      volumeSkipList: [...defaultVolumeSkipList, ...flags['skip-volume']],
       debug: flags.debug,
       userSpecifiedProjectName: flags.project,
       composeFiles: this.config.composeFiles,

packages/core/package.json

@@ -19,6 +19,7 @@
     "jose": "^4.14.4",
     "jsonwebtoken": "^9.0.0",
     "lodash": "^4.17.21",
+    "minimatch": "^9.0.3",
     "node-fetch": "2.6.9",
     "node-forge": "^1.3.1",
     "open": "^8.4.2",

packages/core/src/commands/model.ts

@@ -14,6 +14,7 @@ const composeModel = async ({
   tunnelOpts,
   userSpecifiedProjectName,
   userSpecifiedServices,
+  volumeSkipList,
   scriptInjections,
   composeFiles,
   log,
@@ -33,6 +34,7 @@ const composeModel = async ({
   tunnelOpts: TunnelOpts
   userSpecifiedProjectName: string | undefined
   userSpecifiedServices: string[]
+  volumeSkipList: string[]
   composeFiles: string[]
   log: Logger
   dataDir: string
@@ -55,6 +57,7 @@ const composeModel = async ({
     debug,
     userSpecifiedProjectName,
     userSpecifiedServices,
+    volumeSkipList,
     composeFiles,
     log,
     cwd,

packages/core/src/commands/up.ts

@@ -44,6 +44,7 @@ const up = async ({
   tunnelOpts,
   userSpecifiedProjectName,
   userSpecifiedServices,
+  volumeSkipList,
   scriptInjections,
   composeFiles,
   log,
@@ -67,6 +68,7 @@ const up = async ({
   tunnelOpts: TunnelOpts
   userSpecifiedProjectName: string | undefined
   userSpecifiedServices: string[]
+  volumeSkipList: string[]
   composeFiles: string[]
   log: Logger
   dataDir: string
@@ -87,6 +89,7 @@ const up = async ({
   const {
     model,
     filesToCopy,
+    skippedVolumes,
     projectLocalDataDir,
     createCopiedFile,
   } = await modelCommand({
@@ -98,6 +101,7 @@ const up = async ({
     tunnelOpts,
     userSpecifiedProjectName,
     userSpecifiedServices,
+    volumeSkipList,
     scriptInjections,
     version,
     envId,
@@ -128,6 +132,10 @@ const up = async ({
     })).deployModel
   }
 
+  skippedVolumes.forEach(({ service, source, matchingRule }) => {
+    log.info(`Not copying volume "${source}" for service "${service}" because it matched skip glob "${matchingRule}"`)
+  })
+
   const modelStr = yaml.stringify(composeModel)
   log.debug('model', modelStr)
   const composeFilePath = await createCopiedFile(composeModelFilename, modelStr)

packages/core/src/compose/remote.ts

@@ -1,10 +1,11 @@
 import yaml from 'yaml'
 import path from 'path'
 import { mapValues } from 'lodash'
+import { MMRegExp, makeRe } from 'minimatch'
 import { asyncMap, asyncToArray } from 'iter-tools-es'
 import { COMPOSE_TUNNEL_AGENT_SERVICE_NAME, MachineStatusCommand, ScriptInjection, formatPublicKey } from '@preevy/common'
 import { MachineConnection } from '../driver'
-import { ComposeModel, ComposeSecretOrConfig, composeModelFilename } from './model'
+import { ComposeBindVolume, ComposeModel, ComposeSecretOrConfig, composeModelFilename } from './model'
 import { REMOTE_DIR_BASE, remoteProjectDir } from '../remote-files'
 import { TunnelOpts } from '../ssh'
 import { addComposeTunnelAgentService } from '../compose-tunnel-agent-client'
@@ -28,23 +29,42 @@ const serviceLinkEnvVars = (
     .map(({ name, port, url }) => [`PREEVY_BASE_URI_${name.replace(/[^a-zA-Z0-9_]/g, '_')}_${port}`.toUpperCase(), url])
 )
 
-const volumeSkipList = [
-  /^\/var\/log(\/|$)/,
-  /^\/var\/run(\/|$)/,
-  /^\/$/,
+export const defaultVolumeSkipList: string[] = [
+  '/var/log',
+  '/var/log/**',
+  '/var/run',
+  '/var/run/**',
+  '/',
 ]
 
 const toPosix = (x:string) => x.split(path.sep).join(path.posix.sep)
 
+export type SkippedVolume = { service: string; source: string; matchingRule: string }
+
 const fixModelForRemote = async (
-  { skipServices = [], cwd, remoteBaseDir }: {
+  { skipServices = [], cwd, remoteBaseDir, volumeSkipList = defaultVolumeSkipList }: {
     skipServices?: string[]
     cwd: string
     remoteBaseDir: string
+    volumeSkipList: string[]
   },
   model: ComposeModel,
-): Promise<{ model: Required<Omit<ComposeModel, 'x-preevy' | 'version'>>; filesToCopy: FileToCopy[] }> => {
+): Promise<{
+  model: Required<Omit<ComposeModel, 'x-preevy' | 'version'>>
+  filesToCopy: FileToCopy[]
+  skippedVolumes: SkippedVolume[]
+}> => {
+  const volumeSkipRes = volumeSkipList
+    .map(s => makeRe(path.resolve(cwd, s)))
+    .map((r, i) => {
+      if (!r) {
+        throw new Error(`Invalid glob pattern in volumeSkipList: "${volumeSkipList[i]}"`)
+      }
+      return r as MMRegExp
+    })
+
   const filesToCopy: FileToCopy[] = []
+  const skippedVolumes: SkippedVolume[] = []
 
   const remotePath = (absolutePath: string) => {
     if (!path.isAbsolute(absolutePath)) {
@@ -86,7 +106,9 @@ const fixModelForRemote = async (
         if (volume.type !== 'bind') {
           throw new Error(`Unsupported volume type: ${volume.type} in service ${serviceName}`)
         }
-        if (volumeSkipList.some(re => re.test(volume.source))) {
+        const matchingVolumeSkipIndex = volumeSkipRes.findIndex(re => re.test(volume.source))
+        if (matchingVolumeSkipIndex !== -1) {
+          skippedVolumes.push({ service: serviceName, source: volume.source, matchingRule: volumeSkipList[matchingVolumeSkipIndex] })
           return volume
         }
 
@@ -117,6 +139,7 @@ const fixModelForRemote = async (
       networks: model.networks ?? {},
     },
     filesToCopy,
+    skippedVolumes,
   }
 }
 
@@ -136,6 +159,7 @@ export const remoteComposeModel = async ({
   debug,
   userSpecifiedProjectName,
   userSpecifiedServices,
+  volumeSkipList,
   composeFiles,
   log,
   cwd,
@@ -147,6 +171,7 @@ export const remoteComposeModel = async ({
   debug: boolean
   userSpecifiedProjectName: string | undefined
   userSpecifiedServices: string[]
+  volumeSkipList: string[]
   composeFiles: string[]
   log: Logger
   cwd: string
@@ -172,8 +197,8 @@ export const remoteComposeModel = async ({
     ? [...userSpecifiedServices].concat(COMPOSE_TUNNEL_AGENT_SERVICE_NAME)
     : []
 
-  const { model: fixedModel, filesToCopy } = await fixModelForRemote(
-    { cwd, remoteBaseDir: remoteDir },
+  const { model: fixedModel, filesToCopy, skippedVolumes } = await fixModelForRemote(
+    { cwd, remoteBaseDir: remoteDir, volumeSkipList },
     await modelFilter(await composeClientWithInjectedArgs.getModel(services)),
   )
 
@@ -215,5 +240,5 @@ export const remoteComposeModel = async ({
     filesToCopy.push(sshPrivateKeyFile, knownServerPublicKey)
   }
 
-  return { model, filesToCopy, linkEnvVars }
+  return { model, filesToCopy, skippedVolumes, linkEnvVars }
 }

packages/core/src/index.ts

@@ -27,7 +27,12 @@ export {
 } from './profile'
 export { telemetryEmitter, registerEmitter, wireProcessExit, createTelemetryEmitter, machineId } from './telemetry'
 export { fsTypeFromUrl, Store, VirtualFS, localFsFromUrl, localFs } from './store'
-export { localComposeClient, ComposeModel, resolveComposeFiles, getExposedTcpServicePorts, fetchRemoteUserModel as remoteUserModel, NoComposeFilesError, addScriptInjectionsToServices as addScriptInjectionsToModel } from './compose'
+export {
+  localComposeClient, ComposeModel, resolveComposeFiles, getExposedTcpServicePorts,
+  fetchRemoteUserModel as remoteUserModel, NoComposeFilesError,
+  addScriptInjectionsToServices as addScriptInjectionsToModel,
+  defaultVolumeSkipList,
+} from './compose'
 export { withSpinner } from './spinner'
 export { findEnvId, findProjectName, findEnvIdByProjectName, validateEnvId, normalize as normalizeEnvId, EnvId } from './env-id'
 export { sshKeysStore } from './state'