clean up slim code using glob-all to make it more cross platform (#227)

* basic cleanup aroudn slim option

* clean up slim code using glob-all to make it more cross platform

* fix use of Array.concat

* try omiting \\ in strip command. might need to differ command for windows

* prettier

* use + with -exec for strip bc strip supports multiple files and + doesnt need to be escaped

*  doh prettier again

* damn... find in docker on widnows is finicky

* Update README.md

Author: dschep

README.md

@@ -107,13 +107,15 @@ custom:
 ```  
 #### Custom Removal Patterns  
 To specify additional directories to remove from the installed packages, 
-define the patterns using regex as a `slimPatterns` option in serverless config:  
+define a list of of patterns int he serverless config using the `slimPatterns`
+option and glob syntax. Note, it matches against whole paths, so to match a file in any
+directory, start your pattern with `**/`.
 ```yaml
 custom:
   pythonRequirements:
     slim: true
     slimPatterns:
-      - "*.egg-info*"
+      - "**/*.egg-info*"
 ```  
 This will remove all folders within the installed requirements that match 
 the names in `slimPatterns`  

index.js

@@ -28,6 +28,7 @@ class ServerlessPythonRequirements {
     const options = Object.assign(
       {
         slim: false,
+        slimPatterns: false,
         zip: false,
         cleanupZipHelper: true,
         invalidateCaches: false,

lib/pip.js

@@ -5,7 +5,7 @@ const get = require('lodash.get');
 const set = require('lodash.set');
 const { spawnSync } = require('child_process');
 const { buildImage, getBindPath, getDockerUid } = require('./docker');
-const { getSlimPackageCommands } = require('./slim');
+const { getStripCommand, deleteFiles } = require('./slim');
 
 /**
  * Install requirements described in requirementsPath to targetFolder
@@ -106,12 +106,6 @@ function installRequirements(
     if (process.platform === 'linux') {
       // Use same user so requirements folder is not root and so --cache-dir works
       cmdOptions.push('-u', `${process.getuid()}`);
-      // const stripCmd = quote([
-      //   'find', targetRequirementsFolder,
-      //   '-name', '"*.so"',
-      //   '-exec', 'strip', '{}', '\;',
-      // ]);
-      // pipCmd = ['/bin/bash', '-c', '"' + pipCmd + ' && ' + stripCmd + ' && ' + chownCmd + '"'];
     } else {
       // Use same user so --cache-dir works
       cmdOptions.push('-u', getDockerUid(bindPath));
@@ -123,11 +117,10 @@ function installRequirements(
     cmdOptions = pipCmd.slice(1);
   }
 
-  // If enabled slimming, strip out the caches, tests and dist-infos
+  // If enabled slimming, strip so files
   if (options.slim === true || options.slim === 'true') {
     const preparedPath = dockerPathForWin(options, targetRequirementsFolder);
-    const slimCmd = getSlimPackageCommands(options, preparedPath);
-    cmdOptions.push(...slimCmd);
+    cmdOptions.push(getStripCommand(options, preparedPath));
   }
   let spawnArgs = { cwd: servicePath, shell: true };
   if (process.env.SLS_DEBUG) {
@@ -148,6 +141,10 @@ function installRequirements(
   if (res.status !== 0) {
     throw new Error(res.stderr);
   }
+  // If enabled slimming, delete files in slimPatterns
+  if (options.slim === true || options.slim === 'true') {
+    deleteFiles(options, targetRequirementsFolder);
+  }
 }
 
 /**

lib/slim.js

@@ -1,59 +1,25 @@
 const isWsl = require('is-wsl');
+const glob = require('glob-all');
+const fse = require('fs-extra');
 
-/**
- * Get commands to slim the installed requirements
- * only for non-windows platforms:
- * works for docker builds and when run on UNIX platforms (wsl included)
- * @param {Object} options
- * @param {string} folderPath
- * @return {Array.<String>}
- */
-function getSlimPackageCommands(options, folderPath) {
-  let stripCmd = [];
+const getStripCommand = (options, folderPath) =>
+  process.platform !== 'win32' || isWsl || options.dockerizePip
+    ? ` && find ${folderPath} -name "*.so" -exec strip {} ';'`
+    : '';
 
-  // Default stripping is done for non-windows environments
-  if (process.platform !== 'win32' || isWsl) {
-    stripCmd = getDefaultSLimOptions(folderPath);
-
-    // If specified any custom patterns to remove
-    if (options.slimPatterns instanceof Array) {
-      // Add the custom specified patterns to remove to the default commands
-      const customPatterns = options.slimPatterns.map(pattern => {
-        return getRemovalCommand(folderPath, pattern);
-      });
-      stripCmd = stripCmd.concat(customPatterns);
+const deleteFiles = (options, folderPath) => {
+  let patterns = ['**/*.py[c|o]', '**/__pycache__*', '**/*.dist-info*'];
+  if (options.slimPatterns) {
+    patterns = patterns.concat(options.slimPatterns);
+  }
+  for (const pattern of patterns) {
+    for (const file of glob.sync(`${folderPath}/${pattern}`)) {
+      fse.removeSync(file);
     }
   }
-  return stripCmd;
-}
-
-/**
- * Gets the commands to slim the default (safe) files:
- * including removing caches, stripping compiled files, removing dist-infos
- * @param {String} folderPath
- * @return {Array}
- */
-function getDefaultSLimOptions(folderPath) {
-  return [
-    `&& find ${folderPath} -name "*.so" -exec strip {} \\;`,
-    `&& find ${folderPath} -name "*.py[c|o]" -exec rm -rf {} +`,
-    `&& find ${folderPath} -type d -name "__pycache__*" -exec rm -rf {} +`,
-    `&& find ${folderPath} -type d -name "*.dist-info*" -exec rm -rf {} +`
-  ];
-}
-
-/**
- * Get the command created fromt he find and remove template:
- * returns a string in form `&& find <folder> -name "<match>" -exec rm -rf {} +`
- * @param {String} folderPath
- * @param {String} removalMatch
- * @return {String}
- */
-function getRemovalCommand(folderPath, removalMatch) {
-  return `&& find ${folderPath} -type d -wholename "${removalMatch}" -exec rm -rf {} +`;
-}
+};
 
 module.exports = {
-  getSlimPackageCommands,
-  getDefaultSLimOptions
+  getStripCommand,
+  deleteFiles
 };

tests/base/_slimPatterns.yml

@@ -1,2 +1,2 @@
 slimPatterns:
-  - "*.egg-info*"
+  - "**/*.egg-info*"

tests/pipenv/_slimPatterns.yml

@@ -1,2 +1,2 @@
 slimPatterns:
-  - "*.egg-info*"
+  - "**/*.egg-info*"