Saas 7579 (#514)

oren-codefresh · web-flow · commit 12a9431c414d · 2020-06-23T17:59:11.000+03:00
Download components on image to offline
diff --git a/Dockerfile b/Dockerfile
@@ -39,4 +39,5 @@ RUN apk del yarn
 
 RUN ln -s $(pwd)/lib/interface/cli/codefresh /usr/local/bin/codefresh
 
+RUN codefresh components update --location components
 ENTRYPOINT ["codefresh"]
diff --git a/lib/binary/downloader.js b/lib/binary/downloader.js
@@ -28,28 +28,16 @@ function _ensureDirectory(location) {
 }
 
 async function _getRemoteVersion({
-    name, osType,
+    name, branch, path, file,
 }) {
-    // get the 10 latest releases
-    const releasesUrl = `https://api.github.com/repos/codefresh-io/${name}/releases?per_page=10`;
-    const res = await rp({
-        uri: releasesUrl,
+    const final = branch ? `${name}/${branch}/${path}` : `${name}/${path}`;
+    const url = `https://raw.githubusercontent.com/codefresh-io/${final}/${file}`;
+    const req = await rp({
+        url,
         method: 'GET',
         headers: { 'User-Agent': 'codefresh' },
     });
-
-    const releases = JSON.parse(res);
-    for (let i = 0; i < releases.length; i += 1) {
-        const curRelease = releases[i];
-        const neededAsset = (curRelease.assets || []).find(asset => { 
-            const assetName = asset.name || '';
-            return assetName.startsWith(`${name}_`) && assetName.endsWith(osType);
-        });
-        if (neededAsset) {
-            return { remoteVersion: curRelease.name, download_url: neededAsset.browser_download_url };
-        }
-    }
-    return { remoteVersion: '0', download_url: '' };
+    return req;
 }
 
 function _buildDownloadURL({ name, version, binary }) {
@@ -118,19 +106,26 @@ class Downloader {
 
         let localVersion = _getLocalVersion(join(dir, component.local.versionFile));
         const {
-            repo: name,
+            repo: name, branch, versionPath: path, versionFile: file,
         } = component.remote;
-        let { remoteVersion, download_url } = await _getRemoteVersion({ name, osType });
+        let remoteVersion = await _getRemoteVersion({
+            name, branch, path, file,
+        });
         remoteVersion = remoteVersion.trim();
         localVersion = localVersion.trim();
 
         if (compareVersions(localVersion, remoteVersion) >= 0) {
+            // logger.debug(`Download is not required latest-version=${remoteVersion} local-version=${localVersion}`);
             return Promise.resolve();
         }
-        logger.debug(`${component.name} component upgrade is required, downloading latest version [${remoteVersion}]`);
+        logger.debug(`${component.name} component upgrade is required, downloading.`);
 
-        const resp = await request(download_url);
 
+        const binary = `${name}_${remoteVersion}_${osType}`;
+        const version = component.version.prefix ? `${component.version.prefix}${remoteVersion}` : remoteVersion;
+        const url = _buildDownloadURL({ name, version, binary });
+        const resp = await request(url);
+        
         if (this.progress) {
             let size = 0;
             resp.on('response', (res) => {
@@ -161,11 +156,15 @@ class Downloader {
                 }
                 resolveFn();
             });
+            resp.on('error', (err) => {
+                rejectFn(err);
+            });
         });
     }
 }
 
 module.exports = {
     CommonProgressFormat: 'downloading [{bar}] {percentage}% | {value}/{total}',
     Downloader,
+    CODEFRESH_PATH,
 };
diff --git a/lib/binary/index.js b/lib/binary/index.js
@@ -1,10 +1,11 @@
 const components = require('./components');
 const { Runner } = require('./runner');
-const { Downloader, CommonProgressFormat } = require('./downloader');
+const { Downloader, CommonProgressFormat, CODEFRESH_PATH } = require('./downloader');
 
 module.exports = {
     components,
     Runner,
     Downloader,
     CommonProgressFormat,
+    CODEFRESH_PATH,
 };
diff --git a/lib/interface/cli/commands/cluster/cluster.sdk.spec.js b/lib/interface/cli/commands/cluster/cluster.sdk.spec.js
@@ -1,12 +1,12 @@
 const getCmd = require('./get.cmd').toCommand();
 const deleteCmd = require('./delete.cmd').toCommand();
 const createCmd = require('./create.cmd').toCommand();
-const { sdk } = require('../../../../logic');
-
+const { Runner } = require('./../../../../binary');
 const request = require('requestretry');
 
 jest.mock('../../../../logic/entities/Cluster');
-
+jest.mock('../hybrid/helper');
+jest.mock('./../../../../binary');
 
 const DEFAULT_RESPONSE = request.__defaultResponse();
 
@@ -29,10 +29,9 @@ describe('cluster commands', () => {
 
     describe('create', () => {
         it('should skip', async () => {
-            jest.spyOn(sdk.clusters, 'create').mockImplementation();
             const argv = {};
             await createCmd.handler(argv);
-            expect(sdk.clusters.create).toBeCalled();
+            expect(Runner.mock.instances[0].run).toBeCalled();
         });
     });
 
diff --git a/lib/interface/cli/commands/cluster/create.cmd.js b/lib/interface/cli/commands/cluster/create.cmd.js
@@ -1,8 +1,8 @@
 const Command = require('../../Command');
 const createRoot = require('../root/create.cmd');
 const { sdk } = require('../../../../logic');
-const ProgressEvents = require('../../helpers/progressEvents');
-const cliProgress = require('cli-progress');
+const { downloadSteveDore } = require('../hybrid/helper');
+const { Runner, components } = require('../../../../binary');
 
 const command = new Command({
     command: 'clusters [name]',
@@ -53,33 +53,30 @@ const command = new Command({
         if (terminateProcess === undefined) {
             terminateProcess = true;
         }
-        const events = new ProgressEvents();
-        const format = 'downloading cluster installer  [{bar}] {percentage}% | {value}/{total}';
-        const progressBar = new cliProgress.SingleBar(
-            { stopOnComplete: true, format },
-            cliProgress.Presets.shades_classic,
-        );
-        let clusterTotalSize;
-        events.onStart((size) => {
-            progressBar.start(size, 0);
-            clusterTotalSize = size;
-        });
-        events.onProgress((progress) => {
-            progressBar.update(progress);
-            if (progress >= clusterTotalSize) {
-                console.log('\n');
-            }
-        });
-        return sdk.clusters.create({
+        const binLocation = await downloadSteveDore();
+        const componentRunner = new Runner(binLocation);
+        const commands = [
+            'create',
+            '--c',
             contextName,
-            context,
+            '--token',
+            context.token,
+            '--api-host',
+            `${context.url}/`,
+            '--namespace',
             namespace,
+            '--serviceaccount',
             serviceaccount,
-            behindFirewall,
-            name,
-            terminateProcess,
-            events,
-        });
+        ];
+        if (name) {
+            commands.push('--name-overwrite');
+            commands.push(name);
+        }
+        if (behindFirewall) {
+            commands.push('--behind-firewall');
+        }
+
+        await componentRunner.run(components.stevedore, commands);
     },
 });
 
diff --git a/lib/interface/cli/commands/components/update.cmd.js b/lib/interface/cli/commands/components/update.cmd.js
@@ -0,0 +1,25 @@
+const Command = require('../../Command');
+const componentsRoot = require('../root/components.cmd');
+const helper = require('../hybrid/helper');
+
+const command = new Command({
+    command: 'update',
+    parent: componentsRoot,
+    description: 'Update Codefresh CLI components',
+    webDocs: {
+        category: 'Componenets',
+        title: 'Update',
+    },
+    builder: yargs => yargs
+        .env('CF_ARG_') // this means that every process.env.CF_ARG_* will be passed to argv
+        .option('location', {
+            describe: 'Override dowload folder location',
+        }),
+    handler: async (argv) => {
+        console.log('Updating components');
+        const { location } = argv;
+        await helper.downloadRelatedComponents(location);
+    },
+});
+
+module.exports = command;
diff --git a/lib/interface/cli/commands/hybrid/helper.js b/lib/interface/cli/commands/hybrid/helper.js
@@ -10,9 +10,12 @@ const { followLogs } = require('../../helpers/logs');
 const ProgressEvents = require('../../helpers/progressEvents');
 const cliProgress = require('cli-progress');
 const figlet = require('figlet');
+const path = require('path');
 const {
     components, Runner, Downloader, CommonProgressFormat,
+    CODEFRESH_PATH,
 } = require('./../../../../binary');
+const { pathExists } = require('../../helpers/general');
 
 const INSTALLATION_DEFAULTS = {
     NAMESPACE: 'codefresh',
@@ -23,6 +26,7 @@ const INSTALLATION_DEFAULTS = {
     CF_CONTEXT_NAME: 'cf-runner',
     STORAGE_CLASS_PREFIX: 'dind-local-volumes-runner',
     RESUME_OLD_INSTALLATION: true,
+    COMPONENTS_FOLDER: 'components',
 };
 
 const maxRuntimeNameLength = 63;
@@ -308,7 +312,7 @@ async function getRecommendedKubeNamespace(kubeconfigPath, kubeContextName) {
     return name;
 }
 
-async function _downloadVeonona() {
+async function downloadVeonona(location = CODEFRESH_PATH) {
     const downloader = new Downloader({
         progress: new cliProgress.SingleBar(
             {
@@ -317,13 +321,57 @@ async function _downloadVeonona() {
             },
             cliProgress.Presets.shades_classic,
         ),
+        location,
     });
-    await downloader.download(components.venona);
+    const [error] = await to(downloader.download(components.venona));
+    if (error) {
+        const newLocation = path.join(INSTALLATION_DEFAULTS.COMPONENTS_FOLDER, components.venona.local.dir, components.venona.local.binary);
+        if (await pathExists(newLocation)) {
+            console.log('Failed to download installer, using binary from components folder');
+            return path.resolve(process.cwd(), INSTALLATION_DEFAULTS.COMPONENTS_FOLDER);
+        }
+        console.log('Failed to download component, aborting');
+        throw error;
+    }
+    return location;
+}
+async function downloadSteveDore(location = CODEFRESH_PATH) {
+    const downloader = new Downloader({
+        progress: new cliProgress.SingleBar(
+            {
+                stopOnComplete: true,
+                format: CommonProgressFormat,
+            },
+            cliProgress.Presets.shades_classic,
+        ),
+        location,
+    });
+    const [error] = await to(downloader.download(components.stevedore));
+    if (error) {
+        const newLocation = path.join(
+            INSTALLATION_DEFAULTS.COMPONENTS_FOLDER,
+            components.stevedore.local.dir, components.stevedore.local.binary,
+        );
+        if (await pathExists(newLocation)) {
+            console.log('Failed to download installer, using binary from components folder');
+            return path.resolve(process.cwd(), INSTALLATION_DEFAULTS.COMPONENTS_FOLDER);
+        }
+        console.log('Failed to download component, aborting');
+        throw error;
+    }
+    return location;
+}
+
+async function downloadHybridComponents(location) {
+    await downloadVeonona(location);
+    console.log(`Kubernetes components installer downloaded successfully to ${location} `);
+    await downloadSteveDore(location);
+    console.log(`Kubernetes registrator installer downloaded successfully ${location}`);
 }
 
 async function runClusterAcceptanceTests({ kubeNamespace, kubeConfigPath }) {
-    await _downloadVeonona();
-    const componentRunner = new Runner();
+    const binLocation = await downloadVeonona();
+    const componentRunner = new Runner(binLocation);
     const cmd = ['test', '--log-formtter', DefaultLogFormatter];
     if (kubeNamespace) {
         cmd.push('--kube-namespace');
@@ -353,8 +401,8 @@ async function installAgent({
     verbose, // --verbose
     logFormatting = DefaultLogFormatter, // --log-formtter
 }) {
-    await _downloadVeonona();
-    const componentRunner = new Runner();
+    const binLocation = await downloadVeonona();
+    const componentRunner = new Runner(binLocation);
     const cmd = [
         'install',
         'agent',
@@ -422,8 +470,8 @@ async function installRuntime({
     storageClassName, // --storage-class
     logFormatting = DefaultLogFormatter, // --log-formtter
 }) {
-    await _downloadVeonona();
-    const componentRunner = new Runner();
+    const binLocation = await downloadVeonona();
+    const componentRunner = new Runner(binLocation);
     const cmd = [
         'install',
         'runtime',
@@ -487,8 +535,8 @@ async function attachRuntime({
     runtimeName, // --runtimeName
     logFormatting = DefaultLogFormatter, // --log-formtter
 }) {
-    await _downloadVeonona();
-    const componentRunner = new Runner();
+    const binLocation = await downloadVeonona();
+    const componentRunner = new Runner(binLocation);
     const cmd = [
         'attach',
         '--kube-context-name',
@@ -615,6 +663,9 @@ module.exports = {
     newRuntimeName,
     newAgentName,
     parseNodeSelector,
+    downloadRelatedComponents: downloadHybridComponents,
+    downloadSteveDore,
+    downloadVeonona,
     INSTALLATION_DEFAULTS,
     DefaultLogFormatter,
 };
diff --git a/lib/interface/cli/commands/hybrid/init.cmd.js b/lib/interface/cli/commands/hybrid/init.cmd.js
@@ -111,7 +111,7 @@ const initCmd = new Command({
         .option('install-monitor', {
             describe: 'Install a monitoring component that will help provide valueable data about your cluster to Codefresh',
             type: 'boolean',
-            default: true,            
+            default: true,
         })
         .option('kube-namespace', {
             describe: 'Name of the namespace on which venona should be installed [$CF_ARG_KUBE_NAMESPACE]',
@@ -330,7 +330,6 @@ const initCmd = new Command({
                 installationPlan.addContext('agent', agent);
                 const { token: agentToken } = agent;
                 console.log(`A Codefresh Runner with the name: ${colors.cyan(agentName)} has been created.`);
-                console.log(agentToken);
                 installationPlan.addContext('agentToken', agentToken);
             },
             condition: async () => !installationPlan.getContext('agentToken'),
diff --git a/lib/interface/cli/commands/monitor/install.cmd.js b/lib/interface/cli/commands/monitor/install.cmd.js
diff --git a/lib/interface/cli/commands/root/components.cmd.js b/lib/interface/cli/commands/root/components.cmd.js
diff --git a/package.json b/package.json

Original file line number	Diff line number	Diff line change
`@@ -39,4 +39,5 @@ RUN apk del yarn`
`39`	`39`
`40`	`40`	`RUN ln -s $(pwd)/lib/interface/cli/codefresh /usr/local/bin/codefresh`
`41`	`41`
	`42`	`+RUN codefresh components update --location components`
`42`	`43`	`ENTRYPOINT ["codefresh"]`