// Copyright (c) 2018, Compiler Explorer Authors // All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions are met: // // * Redistributions of source code must retain the above copyright notice, // this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" // AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE // ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE // LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR // CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF // SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS // INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN // CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) // ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE // POSSIBILITY OF SUCH DAMAGE. import path from 'path'; import fs from 'fs-extra'; import _ from 'underscore'; import type {ICompilerArguments, PossibleArguments} from '../types/compiler-arguments.interfaces.js'; import {unwrap} from './assert.js'; import {logger} from './logger.js'; import type {PropertyGetter} from './properties.interfaces.js'; import {S3Bucket} from './s3-handler.js'; import {fileExists, resolvePathFromAppRoot} from './utils.js'; export class CompilerArguments implements ICompilerArguments { private readonly compilerId: string; public possibleArguments: PossibleArguments = {}; private readonly maxPopularArguments = 5; private readonly storeSpecificArguments = false; private loadedFromFile = false; constructor(compilerId: string) { this.compilerId = compilerId; } async loadFromFile(awsProps: PropertyGetter): Promise { let localfolder = awsProps('localfolderArgStats', ''); if (localfolder) { if (localfolder.startsWith('./')) { localfolder = resolvePathFromAppRoot(localfolder); } const filepath = path.join(localfolder, this.compilerId + '.json'); if (await fileExists(filepath)) { const contents = await fs.readFile(filepath); const stats = JSON.parse(contents.toString()); _.each(stats, (times, arg) => { this.addOptionToStatistics(arg, times); }); logger.info(`${this.compilerId}.json loaded from file`); this.loadedFromFile = true; return true; } } return false; } async loadFromStorage(awsProps: PropertyGetter) { if (await this.loadFromFile(awsProps)) return; const region = awsProps('region', ''); const bucket = awsProps('storageBucketArgStats', ''); const prefix = awsProps('storagePrefixArgStats', ''); if (region && bucket && prefix && this.compilerId) { const s3 = new S3Bucket(bucket, region); const result = await s3.get(this.compilerId + '.json', prefix); if (result.hit) { const stats = JSON.parse(unwrap(result.data).toString()); _.each(stats, (times, arg) => { this.addOptionToStatistics(arg, times); }); logger.debug(`${this.compilerId}.json has stats`, stats); } else { logger.debug(`${this.compilerId}.json not present in storage`); } } } getOptimizationArguments(excludeUsedArguments?: string[]): PossibleArguments { excludeUsedArguments = excludeUsedArguments || []; const possibleArguments: PossibleArguments = {}; for (const [argKey, obj] of Object.entries(this.possibleArguments)) { if (!excludeUsedArguments.some(used => this.match(argKey, used))) { if (obj.description.includes('optimize') || obj.description.includes('optimization')) { possibleArguments[argKey] = { description: obj.description, timesused: 0, }; } } } return possibleArguments; } getPopularArguments(excludeUsedArguments?: string[]): PossibleArguments { excludeUsedArguments = excludeUsedArguments || []; const possibleArguments: PossibleArguments = {}; for (const [argKey, obj] of Object.entries(this.possibleArguments)) { if (!excludeUsedArguments.some(used => this.match(argKey, used))) { possibleArguments[argKey] = obj; } } let arr = _.pairs(possibleArguments); arr.sort((a, b) => { if (!this.loadedFromFile && a[1].timesused === 0 && b[1].timesused === 0) { // prefer optimization flags or standard if statistics are not available if (a[1].description.includes('optimization')) { return -1; } else if (b[1].description.includes('optimization')) { return 1; } else if (a[1].description.includes('optimize')) { return -1; } else if (b[1].description.includes('optimize')) { return 1; } else if (a[1].description.includes('std')) { return -1; } else if (b[1].description.includes('std')) { return 1; } } return b[1].timesused - a[1].timesused; }); arr = _.first(arr, this.maxPopularArguments); return _.object(arr); } populateOptions(options: PossibleArguments) { this.possibleArguments = { ...this.possibleArguments, ...options, }; } match(documentedOption: string, givenOption: string): string | boolean { if ( documentedOption.includes('') || documentedOption.includes('') || documentedOption.includes('=val') ) { const numre = /\d*$/i; if (documentedOption.indexOf(givenOption.replace(numre, '')) === 0) { return documentedOption; } } if (documentedOption.includes('=')) { const idx = documentedOption.indexOf('='); if (givenOption.indexOf('=') === idx) { if (documentedOption.substring(0, idx) === givenOption.substring(0, idx)) { return documentedOption; } } } if (documentedOption.includes(':')) { const idx = documentedOption.indexOf(':'); if (givenOption.indexOf(':') === idx) { if (documentedOption.substring(0, idx) === givenOption.substring(0, idx)) { return documentedOption; } } } if (documentedOption.includes('[')) { const idx = documentedOption.indexOf('[') - 1; if (documentedOption.indexOf(givenOption.substring(0, idx)) === 0) { return documentedOption; } } if (documentedOption.indexOf(givenOption) === 0) { return documentedOption; } return false; } addOptionToStatistics(option: string, timesUsed: number | undefined) { if (!timesUsed) timesUsed = 1; const possibleKeys = _.compact(_.keys(this.possibleArguments).map(val => this.match(val, option))) as string[]; for (const key of possibleKeys) { const possibleArgument = this.possibleArguments[key]; if (possibleArgument) { if (possibleKeys.length === 1 || option === key) { possibleArgument.timesused += timesUsed; } else { // non-exact match should be less valuable possibleArgument.timesused += timesUsed - 1; } if (this.storeSpecificArguments && key !== option) { if (!possibleArgument.specifically) { possibleArgument.specifically = []; } possibleArgument.specifically.push({ arg: option, timesused: timesUsed, }); } } } } }