Option to load a custom popular arguments json from disk (#3597)gh-2788

3 files changed, 64 insertions, 23 deletions

diff --git a/etc/config/aws.defaults.properties b/etc/config/aws.defaults.properties
new file mode 100644
index 000000000..7d4de532b
--- /dev/null
+++ b/etc/config/aws.defaults.properties
@@ -0,0 +1 @@
+localfolderArgStats=./compilerargstats
diff --git a/lib/compiler-arguments.js b/lib/compiler-arguments.js
index a87962396..e9261f3ce 100644
--- a/lib/compiler-arguments.js
+++ b/lib/compiler-arguments.js
@@ -22,24 +22,57 @@
 // ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 // POSSIBILITY OF SUCH DAMAGE.
 
+import path from 'path';
+
 import AWS from 'aws-sdk';
+import fs from 'fs-extra';
 import _ from 'underscore';
 
 import {logger} from './logger';
 import {S3Bucket} from './s3-handler';
+import {fileExists, resolvePathFromAppRoot} from './utils';
 
 export class CompilerArguments {
     constructor(compilerId) {
         this.compilerId = compilerId;
-        this.possibleArguments = [];
+        this.possibleArguments = {};
         this.maxPopularArguments = 5;
         this.storeSpecificArguments = false;
+        this.loadedFromFile = false;
+    }
+
+    async loadFromFile(awsProps) {
+        let localfolder = awsProps('localfolderArgStats');
+        if (localfolder) {
+            if (localfolder.startsWith('./')) {
+                localfolder = resolvePathFromAppRoot(localfolder);
+            }
+
+            const filepath = path.join(localfolder, this.compilerId + '.json');
+            if (await fileExists(filepath)) {
+                const contents = await fs.readFile(filepath);
+                const stats = JSON.parse(contents.toString());
+                _.each(stats, (times, arg) => {
+                    this.addOptionToStatistics(arg, times);
+                });
+                logger.info(`${this.compilerId}.json loaded from file`);
+
+                this.loadedFromFile = true;
+
+                return true;
+            }
+        }
+
+        return false;
     }
 
     async loadFromStorage(awsProps) {
+        if (await this.loadFromFile(awsProps)) return;
+
         const region = awsProps('region');
         const bucket = awsProps('storageBucketArgStats');
         const prefix = awsProps('storagePrefixArgStats');
+
         if (region && bucket && this.compilerId) {
             AWS.config.update({region: region});
 
@@ -83,7 +116,7 @@ export class CompilerArguments {
 
         let arr = _.pairs(possibleArguments);
         arr.sort((a, b) => {
-            if (a[1].timesused === 0 && b[1].timesused === 0) {
+            if (!this.loadedFromFile && a[1].timesused === 0 && b[1].timesused === 0) {
                 // prefer optimization flags or standard if statistics are not available
                 if (a[1].description.includes('optimization')) {
                     return -1;
@@ -107,7 +140,10 @@ export class CompilerArguments {
     }
 
     populateOptions(options) {
-        this.possibleArguments = options;
+        this.possibleArguments = {
+            ...this.possibleArguments,
+            ...options,
+        };
     }
 
     match(documentedOption, givenOption) {
@@ -161,7 +197,12 @@ export class CompilerArguments {
 
         for (const key of possibleKeys) {
             if (this.possibleArguments[key]) {
-                this.possibleArguments[key].timesused += timesUsed;
+                if (possibleKeys.length === 1 || option === key) {
+                    this.possibleArguments[key].timesused += timesUsed;
+                } else {
+                    // non-exact match should be less valuable
+                    this.possibleArguments[key].timesused += timesUsed - 1;
+                }
 
                 if (this.storeSpecificArguments && key !== option) {
                     if (!this.possibleArguments[key].specifically) {
diff --git a/lib/compilers/argument-parsers.js b/lib/compilers/argument-parsers.js
index 2ff065a59..1047677ce 100644
--- a/lib/compilers/argument-parsers.js
+++ b/lib/compilers/argument-parsers.js
@@ -24,7 +24,7 @@
 
 import _ from 'underscore';
 
-import { logger } from '../logger';
+import {logger} from '../logger';
 import * as utils from '../utils';
 
 export class BaseParser {
@@ -39,14 +39,13 @@ export class BaseParser {
         utils.eachLine(stdout, line => {
             const match = line.match(optionRegex);
             if (!match) {
-                if (previousOption && (line.trim().length > 0)) {
+                if (previousOption && line.trim().length > 0) {
                     if (options[previousOption].description.endsWith('-'))
                         options[previousOption].description += line.trim();
                     else {
                         if (options[previousOption].description.length > 0)
                             options[previousOption].description += ' ' + line.trim();
-                        else
-                            options[previousOption].description = line.trim();
+                        else options[previousOption].description = line.trim();
                     }
                 } else {
                     previousOption = false;
@@ -69,8 +68,7 @@ export class BaseParser {
     static async getOptions(compiler, helpArg) {
         const optionFinder = /^\s*(--?[\d+,<=>[\]a-z|-]*)\s*(.*)/i;
         const result = await compiler.execCompilerCached(compiler.compiler.exe, [helpArg]);
-        const options = result.code === 0
-            ? BaseParser.parseLines(result.stdout + result.stderr, optionFinder) : {};
+        const options = result.code === 0 ? BaseParser.parseLines(result.stdout + result.stderr, optionFinder) : {};
         compiler.possibleArguments.populateOptions(options);
         return options;
     }
@@ -86,8 +84,11 @@ export class GCCParser extends BaseParser {
         logger.debug(`gcc-like compiler options: ${keys.join(' ')}`);
         if (BaseParser.hasSupport(options, '-masm=')) {
             // -masm= may be available but unsupported by the compiler.
-            const res = await compiler.execCompilerCached(compiler.compiler.exe,
-                ['-fsyntax-only', '--target-help', '-masm=intel']);
+            const res = await compiler.execCompilerCached(compiler.compiler.exe, [
+                '-fsyntax-only',
+                '--target-help',
+                '-masm=intel',
+            ]);
             if (res.code === 0) {
                 compiler.compiler.intelAsm = '-masm=intel';
                 compiler.compiler.supportsIntel = true;
@@ -110,8 +111,10 @@ export class GCCParser extends BaseParser {
 
     static async parse(compiler) {
         const results = await Promise.all([
+            GCCParser.getOptions(compiler, '-fsyntax-only --help'),
             GCCParser.getOptions(compiler, '-fsyntax-only --target-help'),
             GCCParser.getOptions(compiler, '-fsyntax-only --help=common'),
+            GCCParser.getOptions(compiler, '-fsyntax-only --help=warnings'),
             GCCParser.getOptions(compiler, '-fsyntax-only --help=optimizers'),
         ]);
         const options = Object.assign({}, ...results);
@@ -122,8 +125,7 @@ export class GCCParser extends BaseParser {
     static async getOptions(compiler, helpArg) {
         const optionFinder = /^\s*(--?[\d+,<=>[\]a-z|-]*)\s*(.*)/i;
         const result = await compiler.execCompilerCached(compiler.compiler.exe, helpArg.split(' '));
-        const options = result.code === 0
-            ? BaseParser.parseLines(result.stdout + result.stderr, optionFinder) : {};
+        const options = result.code === 0 ? BaseParser.parseLines(result.stdout + result.stderr, optionFinder) : {};
         compiler.possibleArguments.populateOptions(options);
         return options;
     }
@@ -188,8 +190,7 @@ export class ISPCParser extends BaseParser {
     static async getOptions(compiler, helpArg) {
         const result = await compiler.execCompilerCached(compiler.compiler.exe, [helpArg]);
         const optionFinder = /^\s*\[(--?[\d\s()+,/<=>a-z{|}-]*)]\s*(.*)/i;
-        const options = result.code === 0
-            ? BaseParser.parseLines(result.stdout + result.stderr, optionFinder) : {};
+        const options = result.code === 0 ? BaseParser.parseLines(result.stdout + result.stderr, optionFinder) : {};
         compiler.possibleArguments.populateOptions(options);
         return options;
     }
@@ -226,19 +227,18 @@ export class VCParser extends BaseParser {
         let previousOption = false;
         let options = {};
 
-        const matchLine = (line) => {
+        const matchLine = line => {
             if (line.startsWith('/?')) return;
 
             const match = line.match(optionRegex);
             if (!match) {
-                if (previousOption && (line.trim().length > 0)) {
+                if (previousOption && line.trim().length > 0) {
                     if (options[previousOption].description.endsWith(':'))
                         options[previousOption].description += ' ' + line.trim();
                     else {
                         if (options[previousOption].description.length > 0)
                             options[previousOption].description += ', ' + line.trim();
-                        else
-                            options[previousOption].description = line.trim();
+                        else options[previousOption].description = line.trim();
                     }
                 } else {
                     previousOption = false;
@@ -262,7 +262,7 @@ export class VCParser extends BaseParser {
 
             let col1;
             let col2;
-            if ((line.length > 39) && (line[40] === '/')) {
+            if (line.length > 39 && line[40] === '/') {
                 col1 = line.substr(0, 39);
                 col2 = line.substr(40);
             } else {
@@ -280,8 +280,7 @@ export class VCParser extends BaseParser {
     static async getOptions(compiler, helpArg) {
         const result = await compiler.execCompilerCached(compiler.compiler.exe, [helpArg]);
         const optionFinder = /^\s*(\/[\w#+,.:<=>[\]{|}-]*)\s*(.*)/i;
-        const options = result.code === 0
-            ? this.parseLines(result.stdout, optionFinder) : {};
+        const options = result.code === 0 ? this.parseLines(result.stdout, optionFinder) : {};
         compiler.possibleArguments.populateOptions(options);
         return options;
     }