From a7ed1d7945fc4f1f06c8bcbd50049d491681d622 Mon Sep 17 00:00:00 2001 From: Jim Griesmer Date: Fri, 8 Nov 2019 17:13:09 -0800 Subject: [PATCH 01/51] Update .npmrc to get latest python --- .npmrc | 4 +++- package-lock.json | 6 +++--- package.json | 2 +- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/.npmrc b/.npmrc index bc9dcc1dce60..6c091d85dc0a 100644 --- a/.npmrc +++ b/.npmrc @@ -1 +1,3 @@ -@types:registry=https://registry.npmjs.org \ No newline at end of file +@types:registry=https://registry.npmjs.org +@msrvida:registry=https://pkgs.dev.azure.com/msresearch/_packaging/MSR-Python-Analysis/npm/registry/ +always-auth=true diff --git a/package-lock.json b/package-lock.json index 94e3fe286af6..bf7d6068ba60 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1289,9 +1289,9 @@ } }, "@msrvida/python-program-analysis": { - "version": "0.4.1", - "resolved": "https://registry.npmjs.org/@msrvida/python-program-analysis/-/python-program-analysis-0.4.1.tgz", - "integrity": "sha512-8jCtPTTxXyvN3udvz71inScvFvQil7Wh61newdrq79CjuV0W634GCDGuyGpI5G+kgX9PbBZPQJTc1+BMNMB5sQ==" + "version": "0.4.9", + "resolved": "https://pkgs.dev.azure.com/msresearch/_packaging/MSR-Python-Analysis/npm/registry/@msrvida/python-program-analysis/-/python-program-analysis-0.4.9.tgz", + "integrity": "sha1-n3FwxaRKjTYzgMiyTMIUwhDlpbw=" }, "@nteract/markdown": { "version": "3.0.1", diff --git a/package.json b/package.json index 027e5caef809..9f3cfe7dac0e 100644 --- a/package.json +++ b/package.json @@ -2687,7 +2687,7 @@ "@blueprintjs/select": "3.10.0", "@jupyterlab/coreutils": "^3.1.0", "@jupyterlab/services": "^3.2.1", - "@msrvida/python-program-analysis": "^0.4.1", + "@msrvida/python-program-analysis": "^0.4.9", "ansi-regex": "^4.1.0", "arch": "^2.1.0", "azure-storage": "^2.10.3", From bd33869a22cc6ad47cbb70597a7048e8df50a0e9 Mon Sep 17 00:00:00 2001 From: Jim Griesmer Date: Wed, 20 Nov 2019 17:16:52 -0800 Subject: [PATCH 02/51] Update to latest analysis and fix gather icon --- package-lock.json | 6 +++--- package.json | 2 +- .../interactive-common/redux/reducers/helpers.ts | 1 + 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/package-lock.json b/package-lock.json index 68d72a7d56e0..9de4542c4909 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1331,9 +1331,9 @@ } }, "@msrvida/python-program-analysis": { - "version": "0.4.9", - "resolved": "https://pkgs.dev.azure.com/msresearch/_packaging/MSR-Python-Analysis/npm/registry/@msrvida/python-program-analysis/-/python-program-analysis-0.4.9.tgz", - "integrity": "sha1-n3FwxaRKjTYzgMiyTMIUwhDlpbw=" + "version": "0.4.11", + "resolved": "https://pkgs.dev.azure.com/msresearch/_packaging/MSR-Python-Analysis/npm/registry/@msrvida/python-program-analysis/-/python-program-analysis-0.4.11.tgz", + "integrity": "sha1-J+G24XsE+P/eeZMlCCX8O/cfMu8=" }, "@nteract/markdown": { "version": "3.0.1", diff --git a/package.json b/package.json index bb58087036b0..9c6accd21d54 100644 --- a/package.json +++ b/package.json @@ -2694,7 +2694,7 @@ "@blueprintjs/select": "3.10.0", "@jupyterlab/coreutils": "^3.1.0", "@jupyterlab/services": "^4.2.0", - "@msrvida/python-program-analysis": "^0.4.9", + "@msrvida/python-program-analysis": "^0.4.11", "ansi-regex": "^4.1.0", "arch": "^2.1.0", "azure-storage": "^2.10.3", diff --git a/src/datascience-ui/interactive-common/redux/reducers/helpers.ts b/src/datascience-ui/interactive-common/redux/reducers/helpers.ts index b5a452d2a451..24cacf14c236 100644 --- a/src/datascience-ui/interactive-common/redux/reducers/helpers.ts +++ b/src/datascience-ui/interactive-common/redux/reducers/helpers.ts @@ -71,6 +71,7 @@ export namespace Helpers { // and the user has updated the cell text since then. newVMs[index] = { ...newVMs[index], + hasBeenRun: true, cell: { ...newVMs[index].cell, state: arg.payload.state, From b90e5d29ffdebf6432fcc85de37ac24f77b7caae Mon Sep 17 00:00:00 2001 From: Jim Griesmer Date: Wed, 20 Nov 2019 17:57:01 -0800 Subject: [PATCH 03/51] Use sliceLastExecution and fix cell executionId for gather. --- src/client/datascience/gather/gather.ts | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/src/client/datascience/gather/gather.ts b/src/client/datascience/gather/gather.ts index c28000ea5e39..c24786e6c8eb 100644 --- a/src/client/datascience/gather/gather.ts +++ b/src/client/datascience/gather/gather.ts @@ -1,7 +1,7 @@ -import { CellSlice, DataflowAnalyzer, ExecutionLogSlicer } from '@msrvida/python-program-analysis'; -import { Cell as IGatherCell } from '@msrvida/python-program-analysis/dist/es5/cell'; +import { Cell as IGatherCell, CellSlice, DataflowAnalyzer, ExecutionLogSlicer } from '../../../python-program-analysis/src'; import { inject, injectable } from 'inversify'; +import * as uuid from 'uuid/v4'; import { IApplicationShell, ICommandManager } from '../../common/application/types'; import { traceInfo } from '../../common/logger'; import { IConfigurationService, IDisposableRegistry } from '../../common/types'; @@ -62,8 +62,8 @@ export class GatherExecution implements IGatherExecution { const defaultCellMarker = this.configService.getSettings().datascience.defaultCellMarker || Identifiers.DefaultCodeCellMarker; // Call internal slice method - const slices = this._executionSlicer.sliceAllExecutions(gatherCell.persistentId); - const program = slices.length > 0 ? slices[0].cellSlices.reduce(concat, '').replace(/#%%/g, defaultCellMarker) : ''; + const slice = this._executionSlicer.sliceLatestExecution(gatherCell.persistentId); + const program = slice.cellSlices.reduce(concat, '').replace(/#%%/g, defaultCellMarker); // Add a comment at the top of the file explaining what gather does const descriptor = localize.DataScience.gatheredScriptDescription(); @@ -101,7 +101,7 @@ export class GatherExecution implements IGatherExecution { */ function concat(existingText: string, newText: CellSlice): string { // Include our cell marker so that cell slices are preserved - return `${existingText}#%%\n${newText.textSliceLines}\n\n`; + return `${existingText}#%%\n${newText.cell.text}\n`; } /** @@ -115,11 +115,9 @@ function convertVscToGatherCell(cell: IVscCell): IGatherCell | undefined { // tslint:disable-next-line no-unnecessary-local-variable text: cell.data.source, - // This may need to change for native notebook support since in the original Gather code this refers to the number of times that this same cell was executed executionCount: cell.data.execution_count, - executionEventId: cell.id, // This is unique for now, so feed it in + executionEventId: uuid(), - // This may need to change for native notebook support, since this is intended to persist in the metadata for a notebook that is saved and then re-loaded persistentId: cell.id, hasError: cell.state === CellState.error // tslint:disable-next-line: no-any From 2bf3d5843f87fe783011ea7bf0957211bdd18571 Mon Sep 17 00:00:00 2001 From: Jim Griesmer Date: Tue, 26 Nov 2019 22:13:35 -0800 Subject: [PATCH 04/51] Fix import of python analysis --- src/client/datascience/gather/gather.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/client/datascience/gather/gather.ts b/src/client/datascience/gather/gather.ts index c24786e6c8eb..9ba3eb4f3d55 100644 --- a/src/client/datascience/gather/gather.ts +++ b/src/client/datascience/gather/gather.ts @@ -1,4 +1,4 @@ -import { Cell as IGatherCell, CellSlice, DataflowAnalyzer, ExecutionLogSlicer } from '../../../python-program-analysis/src'; +import { Cell as IGatherCell, CellSlice, DataflowAnalyzer, ExecutionLogSlicer } from '@msrvida/python-program-analysis'; import { inject, injectable } from 'inversify'; import * as uuid from 'uuid/v4'; From 699f6ec9ef65eed04b6d9b6c492ae0f66e4ff672 Mon Sep 17 00:00:00 2001 From: Jim Griesmer Date: Fri, 6 Dec 2019 16:46:58 -0800 Subject: [PATCH 05/51] Update to python-analysis 0.4.12 --- package-lock.json | 85 ++++++++++++++++++++++++++++++++++------------- package.json | 2 +- 2 files changed, 62 insertions(+), 25 deletions(-) diff --git a/package-lock.json b/package-lock.json index d03010ff0d8d..4e0c6357aef3 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1331,9 +1331,9 @@ } }, "@msrvida/python-program-analysis": { - "version": "0.4.11", - "resolved": "https://pkgs.dev.azure.com/msresearch/_packaging/MSR-Python-Analysis/npm/registry/@msrvida/python-program-analysis/-/python-program-analysis-0.4.11.tgz", - "integrity": "sha1-J+G24XsE+P/eeZMlCCX8O/cfMu8=" + "version": "0.4.12", + "resolved": "https://pkgs.dev.azure.com/msresearch/_packaging/MSR-Python-Analysis/npm/registry/@msrvida/python-program-analysis/-/python-program-analysis-0.4.12.tgz", + "integrity": "sha1-ic1eFXy+eM1t5xFxdxIDAfSYza0=" }, "@nteract/markdown": { "version": "3.0.1", @@ -4569,7 +4569,8 @@ "version": "2.1.1", "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.1.1.tgz", "integrity": "sha1-w7M6te42DYbg5ijwRorn7yfWVN8=", - "dev": true + "dev": true, + "optional": true }, "aproba": { "version": "1.2.0", @@ -4593,13 +4594,15 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz", "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=", - "dev": true + "dev": true, + "optional": true }, "brace-expansion": { "version": "1.1.11", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", "dev": true, + "optional": true, "requires": { "balanced-match": "^1.0.0", "concat-map": "0.0.1" @@ -4616,19 +4619,22 @@ "version": "1.1.0", "resolved": "https://registry.npmjs.org/code-point-at/-/code-point-at-1.1.0.tgz", "integrity": "sha1-DQcLTQQ6W+ozovGkDi7bPZpMz3c=", - "dev": true + "dev": true, + "optional": true }, "concat-map": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=", - "dev": true + "dev": true, + "optional": true }, "console-control-strings": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/console-control-strings/-/console-control-strings-1.1.0.tgz", "integrity": "sha1-PXz0Rk22RG6mRL9LOVB/mFEAjo4=", - "dev": true + "dev": true, + "optional": true }, "core-util-is": { "version": "1.0.2", @@ -4759,7 +4765,8 @@ "version": "2.0.3", "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz", "integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4=", - "dev": true + "dev": true, + "optional": true }, "ini": { "version": "1.3.5", @@ -4773,6 +4780,7 @@ "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-1.0.0.tgz", "integrity": "sha1-754xOG8DGn8NZDr4L95QxFfvAMs=", "dev": true, + "optional": true, "requires": { "number-is-nan": "^1.0.0" } @@ -4789,6 +4797,7 @@ "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz", "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==", "dev": true, + "optional": true, "requires": { "brace-expansion": "^1.1.7" } @@ -4797,13 +4806,15 @@ "version": "0.0.8", "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.0.8.tgz", "integrity": "sha1-hX/Kv8M5fSYluCKCYuhqp6ARsF0=", - "dev": true + "dev": true, + "optional": true }, "minipass": { "version": "2.3.5", "resolved": "https://registry.npmjs.org/minipass/-/minipass-2.3.5.tgz", "integrity": "sha512-Gi1W4k059gyRbyVUZQ4mEqLm0YIUiGYfvxhF6SIlk3ui1WVxMTGfGdQ2SInh3PDrRTVvPKgULkpJtT4RH10+VA==", "dev": true, + "optional": true, "requires": { "safe-buffer": "^5.1.2", "yallist": "^3.0.0" @@ -4824,6 +4835,7 @@ "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.1.tgz", "integrity": "sha1-MAV0OOrGz3+MR2fzhkjWaX11yQM=", "dev": true, + "optional": true, "requires": { "minimist": "0.0.8" } @@ -4912,7 +4924,8 @@ "version": "1.0.1", "resolved": "https://registry.npmjs.org/number-is-nan/-/number-is-nan-1.0.1.tgz", "integrity": "sha1-CXtgK1NCKlIsGvuHkDGDNpQaAR0=", - "dev": true + "dev": true, + "optional": true }, "object-assign": { "version": "4.1.1", @@ -4926,6 +4939,7 @@ "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", "integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=", "dev": true, + "optional": true, "requires": { "wrappy": "1" } @@ -5021,7 +5035,8 @@ "version": "5.1.2", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", - "dev": true + "dev": true, + "optional": true }, "safer-buffer": { "version": "2.1.2", @@ -5063,6 +5078,7 @@ "resolved": "https://registry.npmjs.org/string-width/-/string-width-1.0.2.tgz", "integrity": "sha1-EYvfW4zcUaKn5w0hHgfisLmxB9M=", "dev": true, + "optional": true, "requires": { "code-point-at": "^1.0.0", "is-fullwidth-code-point": "^1.0.0", @@ -5084,6 +5100,7 @@ "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz", "integrity": "sha1-ajhfuIU9lS1f8F0Oiq+UJ43GPc8=", "dev": true, + "optional": true, "requires": { "ansi-regex": "^2.0.0" } @@ -5132,13 +5149,15 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=", - "dev": true + "dev": true, + "optional": true }, "yallist": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/yallist/-/yallist-3.0.3.tgz", "integrity": "sha512-S+Zk8DEWE6oKpV+vI3qWkaK+jSbIK86pCwe2IF/xwIpQ8jEuxpw9NyaGjmp9+BoJv5FV2piqCDcoCtStppiq2A==", - "dev": true + "dev": true, + "optional": true } } }, @@ -9143,7 +9162,8 @@ "version": "2.1.1", "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.1.1.tgz", "integrity": "sha1-w7M6te42DYbg5ijwRorn7yfWVN8=", - "dev": true + "dev": true, + "optional": true }, "aproba": { "version": "1.2.0", @@ -9165,12 +9185,14 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz", "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=", - "dev": true + "dev": true, + "optional": true }, "brace-expansion": { "version": "1.1.11", "bundled": true, "dev": true, + "optional": true, "requires": { "balanced-match": "^1.0.0", "concat-map": "0.0.1" @@ -9184,19 +9206,22 @@ "version": "1.1.0", "resolved": "https://registry.npmjs.org/code-point-at/-/code-point-at-1.1.0.tgz", "integrity": "sha1-DQcLTQQ6W+ozovGkDi7bPZpMz3c=", - "dev": true + "dev": true, + "optional": true }, "concat-map": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=", - "dev": true + "dev": true, + "optional": true }, "console-control-strings": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/console-control-strings/-/console-control-strings-1.1.0.tgz", "integrity": "sha1-PXz0Rk22RG6mRL9LOVB/mFEAjo4=", - "dev": true + "dev": true, + "optional": true }, "core-util-is": { "version": "1.0.2", @@ -9314,7 +9339,8 @@ "version": "2.0.3", "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz", "integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4=", - "dev": true + "dev": true, + "optional": true }, "ini": { "version": "1.3.5", @@ -9326,6 +9352,7 @@ "version": "1.0.0", "bundled": true, "dev": true, + "optional": true, "requires": { "number-is-nan": "^1.0.0" } @@ -9340,6 +9367,7 @@ "version": "3.0.4", "bundled": true, "dev": true, + "optional": true, "requires": { "brace-expansion": "^1.1.7" } @@ -9348,7 +9376,8 @@ "version": "0.0.8", "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.0.8.tgz", "integrity": "sha1-hX/Kv8M5fSYluCKCYuhqp6ARsF0=", - "dev": true + "dev": true, + "optional": true }, "minipass": { "version": "2.3.5", @@ -9369,6 +9398,7 @@ "version": "0.5.1", "bundled": true, "dev": true, + "optional": true, "requires": { "minimist": "0.0.8" } @@ -9450,7 +9480,8 @@ "version": "1.0.1", "resolved": "https://registry.npmjs.org/number-is-nan/-/number-is-nan-1.0.1.tgz", "integrity": "sha1-CXtgK1NCKlIsGvuHkDGDNpQaAR0=", - "dev": true + "dev": true, + "optional": true }, "object-assign": { "version": "4.1.1", @@ -9462,6 +9493,7 @@ "version": "1.4.0", "bundled": true, "dev": true, + "optional": true, "requires": { "wrappy": "1" } @@ -9582,6 +9614,7 @@ "version": "1.0.2", "bundled": true, "dev": true, + "optional": true, "requires": { "code-point-at": "^1.0.0", "is-fullwidth-code-point": "^1.0.0", @@ -9601,6 +9634,7 @@ "version": "3.0.1", "bundled": true, "dev": true, + "optional": true, "requires": { "ansi-regex": "^2.0.0" } @@ -9638,6 +9672,7 @@ "resolved": "https://registry.npmjs.org/minipass/-/minipass-2.3.5.tgz", "integrity": "sha512-Gi1W4k059gyRbyVUZQ4mEqLm0YIUiGYfvxhF6SIlk3ui1WVxMTGfGdQ2SInh3PDrRTVvPKgULkpJtT4RH10+VA==", "dev": true, + "optional": true, "requires": { "safe-buffer": "^5.1.2", "yallist": "^3.0.0" @@ -9657,7 +9692,8 @@ "version": "5.1.2", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", - "dev": true + "dev": true, + "optional": true } } }, @@ -9680,7 +9716,8 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=", - "dev": true + "dev": true, + "optional": true }, "yallist": { "version": "3.0.3", diff --git a/package.json b/package.json index d37014ab2296..e338b69ddd4c 100644 --- a/package.json +++ b/package.json @@ -2694,7 +2694,7 @@ "@blueprintjs/select": "3.10.0", "@jupyterlab/coreutils": "^3.1.0", "@jupyterlab/services": "^4.2.0", - "@msrvida/python-program-analysis": "^0.4.11", + "@msrvida/python-program-analysis": "^0.4.12", "ansi-regex": "^4.1.0", "arch": "^2.1.0", "azure-storage": "^2.10.3", From e73ec59e0a385b2a39e98ce6716b3373d34957c2 Mon Sep 17 00:00:00 2001 From: Jim Griesmer Date: Tue, 10 Dec 2019 17:57:12 -0800 Subject: [PATCH 06/51] get gathered text from cell.textSlice --- src/client/datascience/gather/gather.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/client/datascience/gather/gather.ts b/src/client/datascience/gather/gather.ts index 9ba3eb4f3d55..b396e54f7e3a 100644 --- a/src/client/datascience/gather/gather.ts +++ b/src/client/datascience/gather/gather.ts @@ -101,7 +101,7 @@ export class GatherExecution implements IGatherExecution { */ function concat(existingText: string, newText: CellSlice): string { // Include our cell marker so that cell slices are preserved - return `${existingText}#%%\n${newText.cell.text}\n`; + return `${existingText}#%%\n${newText.textSlice}\n`; } /** From 6d1475509cbb7991fb02e7318e63121ff2bd24bf Mon Sep 17 00:00:00 2001 From: Jim Griesmer Date: Fri, 13 Dec 2019 17:00:13 -0800 Subject: [PATCH 07/51] Updates to fix gather logging --- src/client/datascience/gather/gather.ts | 2 +- .../datascience/gather/gatherListener.ts | 38 ++----------------- src/client/datascience/gather/gatherLogger.ts | 11 ++++-- .../datascience/jupyter/jupyterNotebook.ts | 9 ++--- .../datascience/jupyter/jupyterServer.ts | 8 ++-- .../jupyter/jupyterServerFactory.ts | 10 ++--- .../jupyter/liveshare/guestJupyterNotebook.ts | 5 --- .../jupyter/liveshare/guestJupyterServer.ts | 4 +- .../jupyter/liveshare/hostJupyterNotebook.ts | 6 +-- .../jupyter/liveshare/hostJupyterServer.ts | 13 ++++--- src/client/datascience/serviceRegistry.ts | 6 ++- src/client/datascience/types.ts | 6 ++- src/test/datascience/execution.unit.test.ts | 31 +++++++-------- 13 files changed, 57 insertions(+), 92 deletions(-) diff --git a/src/client/datascience/gather/gather.ts b/src/client/datascience/gather/gather.ts index b396e54f7e3a..5228911729ba 100644 --- a/src/client/datascience/gather/gather.ts +++ b/src/client/datascience/gather/gather.ts @@ -101,7 +101,7 @@ export class GatherExecution implements IGatherExecution { */ function concat(existingText: string, newText: CellSlice): string { // Include our cell marker so that cell slices are preserved - return `${existingText}#%%\n${newText.textSlice}\n`; + return `${existingText}#%%\n${newText.textSliceLines}\n`; } /** diff --git a/src/client/datascience/gather/gatherListener.ts b/src/client/datascience/gather/gatherListener.ts index 07381f7a423e..1b6ec152106a 100644 --- a/src/client/datascience/gather/gatherListener.ts +++ b/src/client/datascience/gather/gatherListener.ts @@ -10,28 +10,22 @@ import { noop } from '../../common/utils/misc'; import { generateCellsFromString } from '../cellFactory'; import { Identifiers } from '../constants'; import { IInteractiveWindowMapping, InteractiveWindowMessages } from '../interactive-common/interactiveWindowTypes'; -import { ICell, IGatherExecution, IInteractiveWindowListener, IInteractiveWindowProvider, IJupyterExecution, INotebook, INotebookEditorProvider, INotebookExporter } from '../types'; -import { GatherLogger } from './gatherLogger'; +import { ICell, IGatherLogger, IInteractiveWindowListener, INotebookEditorProvider, INotebookExporter } from '../types'; @injectable() export class GatherListener implements IInteractiveWindowListener { // tslint:disable-next-line: no-any private postEmitter: EventEmitter<{ message: string; payload: any }> = new EventEmitter<{ message: string; payload: any }>(); - private gatherLogger: GatherLogger; private notebookUri: Uri | undefined; constructor( - @inject(IGatherExecution) private gather: IGatherExecution, @inject(IApplicationShell) private applicationShell: IApplicationShell, @inject(INotebookExporter) private jupyterExporter: INotebookExporter, @inject(INotebookEditorProvider) private ipynbProvider: INotebookEditorProvider, - @inject(IJupyterExecution) private jupyterExecution: IJupyterExecution, - @inject(IInteractiveWindowProvider) private interactiveWindowProvider: IInteractiveWindowProvider, @inject(IConfigurationService) private configService: IConfigurationService, @inject(IDocumentManager) private documentManager: IDocumentManager, @inject(IFileSystem) private fileSystem: IFileSystem ) { - this.gatherLogger = new GatherLogger(this.gather, this.configService); } public dispose() { @@ -46,16 +40,12 @@ export class GatherListener implements IInteractiveWindowListener { // tslint:disable-next-line: no-any public onMessage(message: string, payload?: any): void { switch (message) { - case InteractiveWindowMessages.NotebookExecutionActivated: - this.handleMessage(message, payload, this.doSetLogger); - break; - case InteractiveWindowMessages.GatherCodeRequest: this.handleMessage(message, payload, this.doGather); break; case InteractiveWindowMessages.RestartKernel: - this.gather.resetLog(); + this.gatherLogger.service().resetLog(); break; default: @@ -69,28 +59,6 @@ export class GatherListener implements IInteractiveWindowListener { handler.bind(this)(args); } - private doSetLogger(payload: string): void { - this.setLogger(payload).ignoreErrors(); - } - - private async setLogger(notebookUri: string) { - this.notebookUri = Uri.parse(notebookUri); - - // First get the active server - const activeServer = await this.jupyterExecution.getServer(await this.interactiveWindowProvider.getNotebookOptions()); - - let nb: INotebook | undefined; - // If that works, see if there's a matching notebook running - if (activeServer) { - nb = await activeServer.getNotebook(this.notebookUri); - - // If we have an executing notebook, add the gather logger. - if (nb) { - nb.addLogger(this.gatherLogger); - } - } - } - private doGather(payload: ICell): void { this.gatherCodeInternal(payload).catch(err => { this.applicationShell.showErrorMessage(err); @@ -98,7 +66,7 @@ export class GatherListener implements IInteractiveWindowListener { } private gatherCodeInternal = async (cell: ICell) => { - const slicedProgram = this.gather.gatherCode(cell); + const slicedProgram = this.gatherLogger.service().gatherCode(cell); if (this.configService.getSettings().datascience.gatherToScript) { await this.showFile(slicedProgram, cell.file); diff --git a/src/client/datascience/gather/gatherLogger.ts b/src/client/datascience/gather/gatherLogger.ts index efae3af72ce0..5c43347226a7 100644 --- a/src/client/datascience/gather/gatherLogger.ts +++ b/src/client/datascience/gather/gatherLogger.ts @@ -5,14 +5,13 @@ import { IConfigurationService } from '../../common/types'; import { noop } from '../../common/utils/misc'; import { CellMatcher } from '../cellMatcher'; import { concatMultilineStringInput } from '../common'; -import { ICell as IVscCell, IGatherExecution, INotebookExecutionLogger } from '../types'; -import { GatherExecution } from './gather'; +import { ICell as IVscCell, IGatherExecution, IGatherLogger, INotebookExecutionLogger } from '../types'; @injectable() -export class GatherLogger implements INotebookExecutionLogger { +export class GatherLogger implements INotebookExecutionLogger, IGatherLogger { constructor( - @inject(GatherExecution) private gather: IGatherExecution, + @inject(IGatherExecution) private gather: IGatherExecution, @inject(IConfigurationService) private configService: IConfigurationService ) { } @@ -38,4 +37,8 @@ export class GatherLogger implements INotebookExecutionLogger { } } } + + public service() { + return this.gather; + } } diff --git a/src/client/datascience/jupyter/jupyterNotebook.ts b/src/client/datascience/jupyter/jupyterNotebook.ts index 8a95705a8505..c26213b4a5ca 100644 --- a/src/client/datascience/jupyter/jupyterNotebook.ts +++ b/src/client/datascience/jupyter/jupyterNotebook.ts @@ -23,6 +23,7 @@ import * as localize from '../../common/utils/localize'; import { noop } from '../../common/utils/misc'; import { StopWatch } from '../../common/utils/stopWatch'; import { PythonInterpreter } from '../../interpreter/contracts'; +import { IServiceContainer } from '../../ioc/types'; import { captureTelemetry, sendTelemetryEvent } from '../../telemetry'; import { generateCells } from '../cellFactory'; import { CellMatcher } from '../cellMatcher'; @@ -154,7 +155,7 @@ export class JupyterNotebookBase implements INotebook { private disposableRegistry: IDisposableRegistry, private owner: INotebookServer, private launchInfo: INotebookServerLaunchInfo, - loggers: INotebookExecutionLogger[], + serviceContainer: IServiceContainer, resource: Uri, private getDisposedError: () => Error, private workspace: IWorkspaceService, @@ -162,7 +163,7 @@ export class JupyterNotebookBase implements INotebook { ) { this.sessionStartTime = Date.now(); this._resource = resource; - this._loggers = [...loggers]; + this._loggers = serviceContainer.getAll(INotebookExecutionLogger); // Save our interpreter and don't change it. Later on when kernel changes // are possible, recompute it. } @@ -264,10 +265,6 @@ export class JupyterNotebookBase implements INotebook { return this.updateWorkingDirectory(file); } - public addLogger(logger: INotebookExecutionLogger) { - this._loggers.push(logger); - } - public executeObservable(code: string, file: string, line: number, id: string, silent: boolean = false): Observable { // Create an observable and wrap the result so we can time it. const stopWatch = new StopWatch(); diff --git a/src/client/datascience/jupyter/jupyterServer.ts b/src/client/datascience/jupyter/jupyterServer.ts index c1541cbfbcf6..670409705ab1 100644 --- a/src/client/datascience/jupyter/jupyterServer.ts +++ b/src/client/datascience/jupyter/jupyterServer.ts @@ -13,13 +13,13 @@ import { IAsyncDisposableRegistry, IConfigurationService, IDisposableRegistry } import { createDeferred, Deferred } from '../../common/utils/async'; import * as localize from '../../common/utils/localize'; import { noop } from '../../common/utils/misc'; +import { IServiceContainer } from '../../ioc/types'; import { IConnection, IJupyterSession, IJupyterSessionManager, IJupyterSessionManagerFactory, INotebook, - INotebookExecutionLogger, INotebookServer, INotebookServerLaunchInfo } from '../types'; @@ -43,7 +43,7 @@ export class JupyterServerBase implements INotebookServer { private disposableRegistry: IDisposableRegistry, private configService: IConfigurationService, private sessionManagerFactory: IJupyterSessionManagerFactory, - private loggers: INotebookExecutionLogger[] + private serviceContainer: IServiceContainer ) { this.asyncRegistry.push(this); } @@ -88,7 +88,7 @@ export class JupyterServerBase implements INotebookServer { this.savedSession = undefined; // Create a notebook and return it. - return this.createNotebookInstance(resource, this.sessionManager, savedSession, this.disposableRegistry, this.configService, this.loggers, cancelToken); + return this.createNotebookInstance(resource, this.sessionManager, savedSession, this.disposableRegistry, this.configService, this.serviceContainer, cancelToken); } public async shutdown(): Promise { @@ -188,7 +188,7 @@ export class JupyterServerBase implements INotebookServer { _savedSession: IJupyterSession | undefined, _disposableRegistry: IDisposableRegistry, _configService: IConfigurationService, - _loggers: INotebookExecutionLogger[], + _serviceContainer: IServiceContainer, _cancelToken?: CancellationToken): Promise { throw new Error('You forgot to override createNotebookInstance'); } diff --git a/src/client/datascience/jupyter/jupyterServerFactory.ts b/src/client/datascience/jupyter/jupyterServerFactory.ts index a6875744c79f..1c898ffd655a 100644 --- a/src/client/datascience/jupyter/jupyterServerFactory.ts +++ b/src/client/datascience/jupyter/jupyterServerFactory.ts @@ -3,7 +3,7 @@ 'use strict'; import '../../common/extensions'; -import { inject, injectable, multiInject, optional } from 'inversify'; +import { inject, injectable } from 'inversify'; import * as uuid from 'uuid/v4'; import { Uri } from 'vscode'; import { CancellationToken } from 'vscode-jsonrpc'; @@ -12,12 +12,12 @@ import * as vsls from 'vsls/vscode'; import { IApplicationShell, ILiveShareApi, IWorkspaceService } from '../../common/application/types'; import { IAsyncDisposableRegistry, IConfigurationService, IDisposableRegistry } from '../../common/types'; import { IInterpreterService } from '../../interpreter/contracts'; +import { IServiceContainer } from '../../ioc/types'; import { IConnection, IDataScience, IJupyterSessionManagerFactory, INotebook, - INotebookExecutionLogger, INotebookServer, INotebookServerLaunchInfo } from '../types'; @@ -38,7 +38,7 @@ type JupyterServerClassType = { configService: IConfigurationService, sessionManager: IJupyterSessionManagerFactory, workspaceService: IWorkspaceService, - loggers: INotebookExecutionLogger[], + serviceContainer: IServiceContainer, appShell: IApplicationShell, interpreterService: IInterpreterService ): IJupyterServerInterface; @@ -60,7 +60,7 @@ export class JupyterServerFactory implements INotebookServer, ILiveShareHasRole @inject(IConfigurationService) configService: IConfigurationService, @inject(IJupyterSessionManagerFactory) sessionManager: IJupyterSessionManagerFactory, @inject(IWorkspaceService) workspaceService: IWorkspaceService, - @multiInject(INotebookExecutionLogger) @optional() loggers: INotebookExecutionLogger[] | undefined, + @inject(IServiceContainer) serviceContainer: IServiceContainer, @inject(IApplicationShell) appShell: IApplicationShell, @inject(IInterpreterService) interpreterService: IInterpreterService) { this.serverFactory = new RoleBasedFactory( @@ -74,7 +74,7 @@ export class JupyterServerFactory implements INotebookServer, ILiveShareHasRole configService, sessionManager, workspaceService, - loggers ? loggers : [], + serviceContainer, appShell, interpreterService ); diff --git a/src/client/datascience/jupyter/liveshare/guestJupyterNotebook.ts b/src/client/datascience/jupyter/liveshare/guestJupyterNotebook.ts index 1a1403b384c0..61c5972a029b 100644 --- a/src/client/datascience/jupyter/liveshare/guestJupyterNotebook.ts +++ b/src/client/datascience/jupyter/liveshare/guestJupyterNotebook.ts @@ -20,7 +20,6 @@ import { IJupyterKernelSpec, INotebook, INotebookCompletion, - INotebookExecutionLogger, INotebookServer, InterruptResult } from '../../types'; @@ -101,10 +100,6 @@ export class GuestJupyterNotebook return Promise.resolve(); } - public addLogger(_logger: INotebookExecutionLogger): void { - noop(); - } - public async setMatplotLibStyle(_useDark: boolean): Promise { // Guest can't change the style. Maybe output a warning here? } diff --git a/src/client/datascience/jupyter/liveshare/guestJupyterServer.ts b/src/client/datascience/jupyter/liveshare/guestJupyterServer.ts index 7c7ad8917bcc..d66d97376472 100644 --- a/src/client/datascience/jupyter/liveshare/guestJupyterServer.ts +++ b/src/client/datascience/jupyter/liveshare/guestJupyterServer.ts @@ -16,7 +16,6 @@ import { IDataScience, IJupyterSessionManagerFactory, INotebook, - INotebookExecutionLogger, INotebookServer, INotebookServerLaunchInfo } from '../../types'; @@ -38,8 +37,7 @@ export class GuestJupyterServer private disposableRegistry: IDisposableRegistry, private configService: IConfigurationService, _sessionManager: IJupyterSessionManagerFactory, - _workspaceService: IWorkspaceService, - _loggers: INotebookExecutionLogger[] + _workspaceService: IWorkspaceService ) { super(liveShare); } diff --git a/src/client/datascience/jupyter/liveshare/hostJupyterNotebook.ts b/src/client/datascience/jupyter/liveshare/hostJupyterNotebook.ts index 0edf833bbf74..6303f0c671f8 100644 --- a/src/client/datascience/jupyter/liveshare/hostJupyterNotebook.ts +++ b/src/client/datascience/jupyter/liveshare/hostJupyterNotebook.ts @@ -14,13 +14,13 @@ import { IApplicationShell, ILiveShareApi, IWorkspaceService } from '../../../co import { traceError } from '../../../common/logger'; import { IConfigurationService, IDisposableRegistry } from '../../../common/types'; import { createDeferred } from '../../../common/utils/async'; +import { IServiceContainer } from '../../../ioc/types'; import { Identifiers, LiveShare, LiveShareCommands } from '../../constants'; import { IExecuteInfo } from '../../interactive-common/interactiveWindowTypes'; import { ICell, IJupyterSession, INotebook, - INotebookExecutionLogger, INotebookServer, INotebookServerLaunchInfo, InterruptResult @@ -48,13 +48,13 @@ export class HostJupyterNotebook disposableRegistry: IDisposableRegistry, owner: INotebookServer, launchInfo: INotebookServerLaunchInfo, - loggers: INotebookExecutionLogger[], + serviceContainer: IServiceContainer, resource: vscode.Uri, getDisposedError: () => Error, workspace: IWorkspaceService, appService: IApplicationShell ) { - super(liveShare, session, configService, disposableRegistry, owner, launchInfo, loggers, resource, getDisposedError, workspace, appService); + super(liveShare, session, configService, disposableRegistry, owner, launchInfo, serviceContainer, resource, getDisposedError, workspace, appService); } public dispose = async (): Promise => { diff --git a/src/client/datascience/jupyter/liveshare/hostJupyterServer.ts b/src/client/datascience/jupyter/liveshare/hostJupyterServer.ts index 82d6c37d6161..8b9a2caa4795 100644 --- a/src/client/datascience/jupyter/liveshare/hostJupyterServer.ts +++ b/src/client/datascience/jupyter/liveshare/hostJupyterServer.ts @@ -13,6 +13,7 @@ import { traceInfo } from '../../../common/logger'; import { IAsyncDisposableRegistry, IConfigurationService, IDisposableRegistry } from '../../../common/types'; import * as localize from '../../../common/utils/localize'; import { StopWatch } from '../../../common/utils/stopWatch'; +import { IServiceContainer } from '../../../ioc/types'; import { sendTelemetryEvent } from '../../../telemetry'; import { Identifiers, LiveShare, LiveShareCommands, RegExpValues, Telemetry } from '../../constants'; import { @@ -21,7 +22,6 @@ import { IJupyterSessionManager, IJupyterSessionManagerFactory, INotebook, - INotebookExecutionLogger, INotebookServer, INotebookServerLaunchInfo } from '../../types'; @@ -47,10 +47,10 @@ export class HostJupyterServer configService: IConfigurationService, sessionManager: IJupyterSessionManagerFactory, private workspaceService: IWorkspaceService, - loggers: INotebookExecutionLogger[], + serviceContainer: IServiceContainer, private appService: IApplicationShell ) { - super(liveShare, asyncRegistry, disposableRegistry, configService, sessionManager, loggers); + super(liveShare, asyncRegistry, disposableRegistry, configService, sessionManager, serviceContainer); } public async dispose(): Promise { @@ -138,7 +138,7 @@ export class HostJupyterServer possibleSession: IJupyterSession | undefined, disposableRegistry: IDisposableRegistry, configService: IConfigurationService, - loggers: INotebookExecutionLogger[], + serviceContainer: IServiceContainer, cancelToken?: CancellationToken): Promise { // See if already exists. @@ -174,11 +174,12 @@ export class HostJupyterServer disposableRegistry, this, launchInfo, - loggers, + serviceContainer, resource, this.getDisposedError.bind(this), this.workspaceService, - this.appService); + this.appService + ); // Wait for it to be ready traceInfo(`Waiting for idle (session) ${this.id}`); diff --git a/src/client/datascience/serviceRegistry.ts b/src/client/datascience/serviceRegistry.ts index 9bc45f03acd6..0c0c904b3182 100644 --- a/src/client/datascience/serviceRegistry.ts +++ b/src/client/datascience/serviceRegistry.ts @@ -60,6 +60,7 @@ import { IDataViewerProvider, IDebugLocationTracker, IGatherExecution, + IGatherLogger, IInteractiveWindow, IInteractiveWindowListener, IInteractiveWindowProvider, @@ -80,6 +81,7 @@ import { IStatusProvider, IThemeFinder } from './types'; +import { GatherLogger } from './gather/gatherLogger'; export function registerTypes(serviceManager: IServiceManager) { serviceManager.addSingleton(IDataScienceCodeLensProvider, DataScienceCodeLensProvider); @@ -113,10 +115,12 @@ export function registerTypes(serviceManager: IServiceManager) { serviceManager.addSingleton(IJupyterDebugger, JupyterDebugger); serviceManager.add(IDataScienceErrorHandler, DataScienceErrorHandler); serviceManager.addSingleton(ICodeLensFactory, CodeLensFactory); - serviceManager.addSingleton(ICellHashProvider, CellHashProvider); + serviceManager.add(ICellHashProvider, CellHashProvider); + serviceManager.add(IGatherLogger, GatherLogger); serviceManager.add(IGatherExecution, GatherExecution); serviceManager.addBinding(ICellHashProvider, IInteractiveWindowListener); serviceManager.addBinding(ICellHashProvider, INotebookExecutionLogger); + serviceManager.addBinding(IGatherLogger, INotebookExecutionLogger); serviceManager.addBinding(IJupyterDebugger, ICellHashListener); serviceManager.addSingleton(INotebookEditorProvider, NativeEditorProvider); serviceManager.add(INotebookEditor, NativeEditor); diff --git a/src/client/datascience/types.ts b/src/client/datascience/types.ts index 241abae05e67..aa6d2a9fce9e 100644 --- a/src/client/datascience/types.ts +++ b/src/client/datascience/types.ts @@ -105,7 +105,6 @@ export interface INotebook extends IAsyncDisposable { setLaunchingFile(file: string): Promise; getSysInfo(): Promise; setMatplotLibStyle(useDark: boolean): Promise; - addLogger(logger: INotebookExecutionLogger): void; getMatchingInterpreter(): PythonInterpreter | undefined; getKernelSpec(): IJupyterKernelSpec | undefined; } @@ -134,6 +133,11 @@ export interface IGatherExecution { resetLog(): void; } +export const IGatherLogger = Symbol('IGatherLogger'); +export interface IGatherLogger { + service(): IGatherExecution; +} + export const IJupyterExecution = Symbol('IJupyterExecution'); export interface IJupyterExecution extends IAsyncDisposable { sessionChanged: Event; diff --git a/src/test/datascience/execution.unit.test.ts b/src/test/datascience/execution.unit.test.ts index af63a769faeb..caa1e91aaef6 100644 --- a/src/test/datascience/execution.unit.test.ts +++ b/src/test/datascience/execution.unit.test.ts @@ -53,7 +53,6 @@ import { IJupyterKernelSpec, INotebook, INotebookCompletion, - INotebookExecutionLogger, INotebookServer, INotebookServerLaunchInfo, InterruptResult @@ -112,10 +111,6 @@ class MockJupyterNotebook implements INotebook { noop(); } - public addLogger(_logger: INotebookExecutionLogger): void { - noop(); - } - public getSysInfo(): Promise { return Promise.resolve(undefined); } @@ -454,7 +449,7 @@ suite('Jupyter Execution', async () => { .returns(() => result); } - function createKernelSpecs(specs: {name: string; resourceDir: string}[]): Record { + function createKernelSpecs(specs: { name: string; resourceDir: string }[]): Record { const models: Record = {}; specs.forEach(spec => { models[spec.name] = { @@ -479,18 +474,18 @@ suite('Jupyter Execution', async () => { setupPythonServiceWithFunc(service, 'jupyter', ['kernelspec', 'list', '--json'], () => { // Return different results after we install our kernel if (ipykernelInstallCount > 0) { - const kernelSpecs = createKernelSpecs([{name: 'working', resourceDir: path.dirname(workingKernelSpec)}, {name: '0e8519db-0895-416c-96df-fa80131ecea0', resourceDir: 'C:\\Users\\rchiodo\\AppData\\Roaming\\jupyter\\kernels\\0e8519db-0895-416c-96df-fa80131ecea0'}]); + const kernelSpecs = createKernelSpecs([{ name: 'working', resourceDir: path.dirname(workingKernelSpec) }, { name: '0e8519db-0895-416c-96df-fa80131ecea0', resourceDir: 'C:\\Users\\rchiodo\\AppData\\Roaming\\jupyter\\kernels\\0e8519db-0895-416c-96df-fa80131ecea0' }]); return Promise.resolve({ stdout: JSON.stringify(kernelSpecs) }); } else { - const kernelSpecs = createKernelSpecs([{name: '0e8519db-0895-416c-96df-fa80131ecea0', resourceDir: 'C:\\Users\\rchiodo\\AppData\\Roaming\\jupyter\\kernels\\0e8519db-0895-416c-96df-fa80131ecea0'}]); + const kernelSpecs = createKernelSpecs([{ name: '0e8519db-0895-416c-96df-fa80131ecea0', resourceDir: 'C:\\Users\\rchiodo\\AppData\\Roaming\\jupyter\\kernels\\0e8519db-0895-416c-96df-fa80131ecea0' }]); return Promise.resolve({ stdout: JSON.stringify(kernelSpecs) }); } }); - const kernelSpecs2 = createKernelSpecs([{name: 'working', resourceDir: path.dirname(workingKernelSpec)}, {name: '0e8519db-0895-416c-96df-fa80131ecea0', resourceDir: 'C:\\Users\\rchiodo\\AppData\\Roaming\\jupyter\\kernels\\0e8519db-0895-416c-96df-fa80131ecea0'}]); + const kernelSpecs2 = createKernelSpecs([{ name: 'working', resourceDir: path.dirname(workingKernelSpec) }, { name: '0e8519db-0895-416c-96df-fa80131ecea0', resourceDir: 'C:\\Users\\rchiodo\\AppData\\Roaming\\jupyter\\kernels\\0e8519db-0895-416c-96df-fa80131ecea0' }]); setupPythonService(service, 'jupyter', ['kernelspec', 'list', '--json'], Promise.resolve({ stdout: JSON.stringify(kernelSpecs2) })); setupPythonServiceWithFunc(service, 'ipykernel', ['install', '--user', '--name', /\w+-\w+-\w+-\w+-\w+/, '--display-name', `'Python Interactive'`], () => { ipykernelInstallCount += 1; - const kernelSpecs = createKernelSpecs([{name: 'somename', resourceDir: path.dirname(workingKernelSpec)}]); + const kernelSpecs = createKernelSpecs([{ name: 'somename', resourceDir: path.dirname(workingKernelSpec) }]); return Promise.resolve({ stdout: JSON.stringify(kernelSpecs) }); }); const getServerInfoPath = path.join(EXTENSION_ROOT_DIR, 'pythonFiles', 'datascience', 'getServerInfo.py'); @@ -504,7 +499,7 @@ suite('Jupyter Execution', async () => { setupPythonService(service, 'jupyter', ['notebook', '--version'], Promise.resolve({ stdout: '1.1.1.1' })); setupPythonService(service, 'jupyter', ['kernelspec', '--version'], Promise.resolve({ stdout: '1.1.1.1' })); service.setup(x => x.getInterpreterInformation()).returns(() => Promise.resolve(missingKernelPython)); - const kernelSpecs = createKernelSpecs([{name: 'working', resourceDir: path.dirname(workingKernelSpec)}]); + const kernelSpecs = createKernelSpecs([{ name: 'working', resourceDir: path.dirname(workingKernelSpec) }]); setupPythonService(service, 'jupyter', ['kernelspec', 'list', '--json'], Promise.resolve({ stdout: JSON.stringify(kernelSpecs) })); const getServerInfoPath = path.join(EXTENSION_ROOT_DIR, 'pythonFiles', 'datascience', 'getServerInfo.py'); setupPythonService(service, undefined, [getServerInfoPath], Promise.resolve({ stdout: 'failure to get server infos' })); @@ -525,18 +520,18 @@ suite('Jupyter Execution', async () => { setupProcessServiceExecWithFunc(service, workingPython.path, ['-m', 'jupyter', 'kernelspec', 'list', '--json'], () => { // Return different results after we install our kernel if (ipykernelInstallCount > 0) { - const kernelSpecs = createKernelSpecs([{name: 'working', resourceDir: path.dirname(workingKernelSpec)}, {name: '0e8519db-0895-416c-96df-fa80131ecea0', resourceDir: 'C:\\Users\\rchiodo\\AppData\\Roaming\\jupyter\\kernels\\0e8519db-0895-416c-96df-fa80131ecea0'}]); + const kernelSpecs = createKernelSpecs([{ name: 'working', resourceDir: path.dirname(workingKernelSpec) }, { name: '0e8519db-0895-416c-96df-fa80131ecea0', resourceDir: 'C:\\Users\\rchiodo\\AppData\\Roaming\\jupyter\\kernels\\0e8519db-0895-416c-96df-fa80131ecea0' }]); return Promise.resolve({ stdout: JSON.stringify(kernelSpecs) }); } else { - const kernelSpecs = createKernelSpecs([{name: '0e8519db-0895-416c-96df-fa80131ecea0', resourceDir: 'C:\\Users\\rchiodo\\AppData\\Roaming\\jupyter\\kernels\\0e8519db-0895-416c-96df-fa80131ecea0'}]); + const kernelSpecs = createKernelSpecs([{ name: '0e8519db-0895-416c-96df-fa80131ecea0', resourceDir: 'C:\\Users\\rchiodo\\AppData\\Roaming\\jupyter\\kernels\\0e8519db-0895-416c-96df-fa80131ecea0' }]); return Promise.resolve({ stdout: JSON.stringify(kernelSpecs) }); } }); - const kernelSpecs2 = createKernelSpecs([{name: 'working', resourceDir: path.dirname(workingKernelSpec)}, {name: '0e8519db-0895-416c-96df-fa80131ecea0', resourceDir: 'C:\\Users\\rchiodo\\AppData\\Roaming\\jupyter\\kernels\\0e8519db-0895-416c-96df-fa80131ecea0'}]); - setupProcessServiceExec(service, workingPython.path, ['-m', 'jupyter', 'kernelspec', 'list', '--json'], Promise.resolve({ stdout: JSON.stringify(kernelSpecs2)})); + const kernelSpecs2 = createKernelSpecs([{ name: 'working', resourceDir: path.dirname(workingKernelSpec) }, { name: '0e8519db-0895-416c-96df-fa80131ecea0', resourceDir: 'C:\\Users\\rchiodo\\AppData\\Roaming\\jupyter\\kernels\\0e8519db-0895-416c-96df-fa80131ecea0' }]); + setupProcessServiceExec(service, workingPython.path, ['-m', 'jupyter', 'kernelspec', 'list', '--json'], Promise.resolve({ stdout: JSON.stringify(kernelSpecs2) })); setupProcessServiceExecWithFunc(service, workingPython.path, ['-m', 'ipykernel', 'install', '--user', '--name', /\w+-\w+-\w+-\w+-\w+/, '--display-name', `'Python Interactive'`], () => { ipykernelInstallCount += 1; - const kernelSpecs = createKernelSpecs([{name: 'somename', resourceDir: path.dirname(workingKernelSpec)}]); + const kernelSpecs = createKernelSpecs([{ name: 'somename', resourceDir: path.dirname(workingKernelSpec) }]); return Promise.resolve({ stdout: JSON.stringify(kernelSpecs) }); }); const getServerInfoPath = path.join(EXTENSION_ROOT_DIR, 'pythonFiles', 'datascience', 'getServerInfo.py'); @@ -546,7 +541,7 @@ suite('Jupyter Execution', async () => { } function setupMissingKernelProcessService(service: TypeMoq.IMock, notebookStdErr?: string[]) { - const kernelSpecs = createKernelSpecs([{name: 'working', resourceDir: path.dirname(workingKernelSpec)}]); + const kernelSpecs = createKernelSpecs([{ name: 'working', resourceDir: path.dirname(workingKernelSpec) }]); setupProcessServiceExec(service, missingKernelPython.path, ['-m', 'jupyter', 'kernelspec', 'list', '--json'], Promise.resolve({ stdout: JSON.stringify(kernelSpecs) })); const getServerInfoPath = path.join(EXTENSION_ROOT_DIR, 'pythonFiles', 'datascience', 'getServerInfo.py'); setupProcessServiceExec(service, missingKernelPython.path, [getServerInfoPath], Promise.resolve({ stdout: 'failure to get server infos' })); @@ -555,7 +550,7 @@ suite('Jupyter Execution', async () => { } function setupPathProcessService(jupyterPath: string, service: TypeMoq.IMock, notebookStdErr?: string[]) { - const kernelSpecs = createKernelSpecs([{name: 'working', resourceDir: path.dirname(workingKernelSpec)}]); + const kernelSpecs = createKernelSpecs([{ name: 'working', resourceDir: path.dirname(workingKernelSpec) }]); setupProcessServiceExec(service, jupyterPath, ['kernelspec', 'list', '--json'], Promise.resolve({ stdout: JSON.stringify(kernelSpecs) })); setupProcessServiceExecObservable(service, jupyterPath, ['kernelspec', 'list', '--json'], [], []); setupProcessServiceExec(service, jupyterPath, ['--version'], Promise.resolve({ stdout: '1.1.1.1' })); From 6e875700ad72e28baa9138f419b7ccbb32565e22 Mon Sep 17 00:00:00 2001 From: Jim Griesmer Date: Mon, 16 Dec 2019 13:29:00 -0800 Subject: [PATCH 08/51] All updates to using new nb owned gather. --- .../datascience/gather/gatherListener.ts | 37 +++++++++++++++++-- .../datascience/jupyter/jupyterNotebook.ts | 7 +++- .../jupyter/liveshare/guestJupyterNotebook.ts | 5 +++ src/client/datascience/types.ts | 3 +- src/test/datascience/execution.unit.test.ts | 5 +++ 5 files changed, 52 insertions(+), 5 deletions(-) diff --git a/src/client/datascience/gather/gatherListener.ts b/src/client/datascience/gather/gatherListener.ts index 1b6ec152106a..25fc471f0ff5 100644 --- a/src/client/datascience/gather/gatherListener.ts +++ b/src/client/datascience/gather/gatherListener.ts @@ -10,19 +10,22 @@ import { noop } from '../../common/utils/misc'; import { generateCellsFromString } from '../cellFactory'; import { Identifiers } from '../constants'; import { IInteractiveWindowMapping, InteractiveWindowMessages } from '../interactive-common/interactiveWindowTypes'; -import { ICell, IGatherLogger, IInteractiveWindowListener, INotebookEditorProvider, INotebookExporter } from '../types'; +import { ICell, IGatherExecution, IInteractiveWindowListener, IInteractiveWindowProvider, IJupyterExecution, INotebook, INotebookEditorProvider, INotebookExporter } from '../types'; @injectable() export class GatherListener implements IInteractiveWindowListener { // tslint:disable-next-line: no-any private postEmitter: EventEmitter<{ message: string; payload: any }> = new EventEmitter<{ message: string; payload: any }>(); private notebookUri: Uri | undefined; + private gatherService: IGatherExecution | undefined; constructor( @inject(IApplicationShell) private applicationShell: IApplicationShell, @inject(INotebookExporter) private jupyterExporter: INotebookExporter, @inject(INotebookEditorProvider) private ipynbProvider: INotebookEditorProvider, @inject(IConfigurationService) private configService: IConfigurationService, + @inject(IJupyterExecution) private jupyterExecution: IJupyterExecution, + @inject(IInteractiveWindowProvider) private interactiveWindowProvider: IInteractiveWindowProvider, @inject(IDocumentManager) private documentManager: IDocumentManager, @inject(IFileSystem) private fileSystem: IFileSystem ) { @@ -40,12 +43,18 @@ export class GatherListener implements IInteractiveWindowListener { // tslint:disable-next-line: no-any public onMessage(message: string, payload?: any): void { switch (message) { + case InteractiveWindowMessages.NotebookExecutionActivated: + this.handleMessage(message, payload, this.doInitGather); + break; + case InteractiveWindowMessages.GatherCodeRequest: this.handleMessage(message, payload, this.doGather); break; case InteractiveWindowMessages.RestartKernel: - this.gatherLogger.service().resetLog(); + if (this.gatherService) { + this.gatherService.resetLog(); + } break; default: @@ -65,8 +74,30 @@ export class GatherListener implements IInteractiveWindowListener { }); } + private doInitGather(payload: string): void { + this.initGather(payload).ignoreErrors(); + } + + private async initGather(notebookUri: string) { + this.notebookUri = Uri.parse(notebookUri); + + // First get the active server + const activeServer = await this.jupyterExecution.getServer(await this.interactiveWindowProvider.getNotebookOptions()); + + let nb: INotebook | undefined; + // If that works, see if there's a matching notebook running + if (activeServer) { + nb = await activeServer.getNotebook(this.notebookUri); + + // If we have an executing notebook, get its gather execution service. + if (nb) { + this.gatherService = nb.getGatherService(); + } + } + } + private gatherCodeInternal = async (cell: ICell) => { - const slicedProgram = this.gatherLogger.service().gatherCode(cell); + const slicedProgram = (this.gatherService) ? this.gatherService.gatherCode(cell) : ''; if (this.configService.getSettings().datascience.gatherToScript) { await this.showFile(slicedProgram, cell.file); diff --git a/src/client/datascience/jupyter/jupyterNotebook.ts b/src/client/datascience/jupyter/jupyterNotebook.ts index 4e0332616fa6..573f55fc68ec 100644 --- a/src/client/datascience/jupyter/jupyterNotebook.ts +++ b/src/client/datascience/jupyter/jupyterNotebook.ts @@ -25,7 +25,7 @@ import { generateCells } from '../cellFactory'; import { CellMatcher } from '../cellMatcher'; import { concatMultilineStringInput, concatMultilineStringOutput, formatStreamText } from '../common'; import { CodeSnippits, Identifiers, Telemetry } from '../constants'; -import { CellState, ICell, IJupyterKernelSpec, IJupyterSession, INotebook, INotebookCompletion, INotebookExecutionLogger, INotebookServer, INotebookServerLaunchInfo, InterruptResult } from '../types'; +import { CellState, ICell, IGatherExecution, IGatherLogger, IJupyterKernelSpec, IJupyterSession, INotebook, INotebookCompletion, INotebookExecutionLogger, INotebookServer, INotebookServerLaunchInfo, InterruptResult } from '../types'; import { expandWorkingDir } from './jupyterUtils'; // tslint:disable-next-line: no-require-imports @@ -512,6 +512,11 @@ export class JupyterNotebookBase implements INotebook { } } + public getGatherService(): IGatherExecution | undefined { + const gatherLogger: INotebookExecutionLogger | undefined = this._loggers.find((logger: INotebookExecutionLogger) => { return (logger).service !== undefined; }); + return gatherLogger ? (gatherLogger).service() : undefined; + } + private async initializeMatplotlib(cancelToken?: CancellationToken): Promise { const settings = this.configService.getSettings().datascience; if (settings && settings.themeMatplotlibPlots) { diff --git a/src/client/datascience/jupyter/liveshare/guestJupyterNotebook.ts b/src/client/datascience/jupyter/liveshare/guestJupyterNotebook.ts index 1146fd7105ba..cb401e9d7899 100644 --- a/src/client/datascience/jupyter/liveshare/guestJupyterNotebook.ts +++ b/src/client/datascience/jupyter/liveshare/guestJupyterNotebook.ts @@ -17,6 +17,7 @@ import { PythonInterpreter } from '../../../interpreter/contracts'; import { LiveShare, LiveShareCommands } from '../../constants'; import { ICell, + IGatherExecution, IJupyterKernelSpec, INotebook, INotebookCompletion, @@ -200,6 +201,10 @@ export class GuestJupyterNotebook return; } + public getGatherService(): IGatherExecution | undefined { + return; + } + public setKernelSpec(_spec: IJupyterKernelSpec): Promise { return Promise.resolve(); } diff --git a/src/client/datascience/types.ts b/src/client/datascience/types.ts index 41a6b598bbf7..35432c628eb5 100644 --- a/src/client/datascience/types.ts +++ b/src/client/datascience/types.ts @@ -103,6 +103,7 @@ export interface INotebook extends IAsyncDisposable { getKernelSpec(): IJupyterKernelSpec | undefined; setKernelSpec(spec: IJupyterKernelSpec): Promise; setInterpreter(interpeter: PythonInterpreter): void; + getGatherService(): IGatherExecution | undefined; } export interface INotebookServerOptions { @@ -130,7 +131,7 @@ export interface IGatherExecution { } export const IGatherLogger = Symbol('IGatherLogger'); -export interface IGatherLogger { +export interface IGatherLogger extends INotebookExecutionLogger { service(): IGatherExecution; } diff --git a/src/test/datascience/execution.unit.test.ts b/src/test/datascience/execution.unit.test.ts index 8f22f5e35eca..17639a9d3c61 100644 --- a/src/test/datascience/execution.unit.test.ts +++ b/src/test/datascience/execution.unit.test.ts @@ -50,6 +50,7 @@ import { NotebookStarter } from '../../client/datascience/jupyter/notebookStarte import { ICell, IConnection, + IGatherExecution, IJupyterKernelSpec, INotebook, INotebookCompletion, @@ -144,6 +145,10 @@ class MockJupyterNotebook implements INotebook { return Promise.resolve(); } + public getGatherService(): IGatherExecution | undefined { + return; + } + public get onSessionStatusChanged(): Event { if (!this.onStatusChangedEvent) { this.onStatusChangedEvent = new EventEmitter(); From 811fb97e53321768c2d2b01cf8bc43a6b4c042be Mon Sep 17 00:00:00 2001 From: Jim Griesmer Date: Fri, 17 Jan 2020 17:19:57 -0800 Subject: [PATCH 09/51] Working cell execution count codelenses. --- package-lock.json | 2 +- package.json | 2 +- .../editor-integration/cellhashLogger.ts | 48 +++++ .../editor-integration/cellhashprovider.ts | 173 ++++++++---------- .../editor-integration/codeLensFactory.ts | 54 +++++- .../datascience/gather/gatherListener.ts | 5 +- src/client/datascience/gather/gatherLogger.ts | 11 +- .../datascience/jupyter/jupyterNotebook.ts | 31 +++- .../jupyter/liveshare/guestJupyterNotebook.ts | 6 +- src/client/datascience/serviceRegistry.ts | 14 +- src/client/datascience/types.ts | 6 + .../cellhashprovider.unit.test.ts | 5 +- .../codewatcher.unit.test.ts | 11 +- src/test/datascience/execution.unit.test.ts | 5 + 14 files changed, 231 insertions(+), 142 deletions(-) create mode 100644 src/client/datascience/editor-integration/cellhashLogger.ts diff --git a/package-lock.json b/package-lock.json index 266f89cb8fd7..b80d82bd20ae 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "python", - "version": "2020.1.1-dev", + "version": "2020.2.0-dev", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/package.json b/package.json index 0e297eed9e9d..908a439616b8 100644 --- a/package.json +++ b/package.json @@ -1838,7 +1838,7 @@ "query": "whos", "parseExpr": "'(\\w+)'" } - ] + ] ] }, "python.disableInstallationCheck": { diff --git a/src/client/datascience/editor-integration/cellhashLogger.ts b/src/client/datascience/editor-integration/cellhashLogger.ts new file mode 100644 index 000000000000..94aab9971af7 --- /dev/null +++ b/src/client/datascience/editor-integration/cellhashLogger.ts @@ -0,0 +1,48 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +'use strict'; +import { inject, injectable } from 'inversify'; + +import { traceError } from '../../common/logger'; +import { noop } from '../../common/utils/misc'; +import { Identifiers } from '../constants'; +import { ICell, ICellHashLogger, ICellHashProvider } from '../types'; +import { CellHashProvider } from './cellhashprovider'; + +// This class provides hashes for debugging jupyter cells. Call getHashes just before starting debugging to compute all of the +// hashes for cells. +@injectable() +export class CellHashLogger implements ICellHashLogger { + constructor(@inject(ICellHashProvider) private provider: ICellHashProvider) { } + + public async preExecute(cell: ICell, silent: boolean): Promise { + const providerObj: CellHashProvider = this.provider as CellHashProvider; + + try { + if (!silent) { + // Don't log empty cells + const stripped = providerObj.extractExecutableLines(cell.data.source); + if (stripped.length > 0 && stripped.find(s => s.trim().length > 0)) { + // When the user adds new code, we know the execution count is increasing + providerObj.executionCount += 1; + + // Skip hash on unknown file though + if (cell.file !== Identifiers.EmptyFileName) { + await providerObj.addCellHash(cell, providerObj.executionCount); + } + } + } + } catch (exc) { + // Don't let exceptions in a preExecute mess up normal operation + traceError(exc); + } + } + + public async postExecute(_cell: ICell, _silent: boolean): Promise { + noop(); + } + + public getProvider(): ICellHashProvider { + return this.provider; + } +} diff --git a/src/client/datascience/editor-integration/cellhashprovider.ts b/src/client/datascience/editor-integration/cellhashprovider.ts index e7968981dfc9..8a3488941d3a 100644 --- a/src/client/datascience/editor-integration/cellhashprovider.ts +++ b/src/client/datascience/editor-integration/cellhashprovider.ts @@ -7,15 +7,13 @@ import { inject, injectable, multiInject, optional } from 'inversify'; import { Event, EventEmitter, Position, Range, TextDocumentChangeEvent, TextDocumentContentChangeEvent } from 'vscode'; import { IDebugService, IDocumentManager } from '../../common/application/types'; -import { traceError, traceInfo } from '../../common/logger'; +import { traceInfo } from '../../common/logger'; import { IFileSystem } from '../../common/platform/types'; import { IConfigurationService } from '../../common/types'; -import { noop } from '../../common/utils/misc'; import { CellMatcher } from '../cellMatcher'; import { splitMultilineString } from '../common'; -import { Identifiers } from '../constants'; import { InteractiveWindowMessages, SysInfoReason } from '../interactive-common/interactiveWindowTypes'; -import { ICell, ICellHash, ICellHashListener, ICellHashProvider, IFileHashes, IInteractiveWindowListener, INotebookExecutionLogger } from '../types'; +import { ICell, ICellHash, ICellHashListener, ICellHashProvider, IFileHashes, IInteractiveWindowListener } from '../types'; interface IRangedCellHash extends ICellHash { code: string; @@ -28,12 +26,21 @@ interface IRangedCellHash extends ICellHash { // This class provides hashes for debugging jupyter cells. Call getHashes just before starting debugging to compute all of the // hashes for cells. @injectable() -export class CellHashProvider implements ICellHashProvider, IInteractiveWindowListener, INotebookExecutionLogger { +export class CellHashProvider implements ICellHashProvider, IInteractiveWindowListener { + public get updated(): Event { + return this.updateEventEmitter.event; + } + + // tslint:disable-next-line: no-any + public get postMessage(): Event<{ message: string; payload: any }> { + return this.postEmitter.event; + } + public executionCount: number = 0; + // tslint:disable-next-line: no-any private postEmitter: EventEmitter<{ message: string; payload: any }> = new EventEmitter<{ message: string; payload: any }>(); // Map of file to Map of start line to actual hash private hashes: Map = new Map(); - private executionCount: number = 0; private updateEventEmitter: EventEmitter = new EventEmitter(); constructor( @@ -51,15 +58,6 @@ export class CellHashProvider implements ICellHashProvider, IInteractiveWindowLi this.hashes.clear(); } - public get updated(): Event { - return this.updateEventEmitter.event; - } - - // tslint:disable-next-line: no-any - public get postMessage(): Event<{ message: string; payload: any }> { - return this.postEmitter.event; - } - // tslint:disable-next-line: no-any public onMessage(message: string, payload?: any): void { switch (message) { @@ -90,91 +88,7 @@ export class CellHashProvider implements ICellHashProvider, IInteractiveWindowLi .filter(e => e.hashes.length > 0); } - public async preExecute(cell: ICell, silent: boolean): Promise { - try { - if (!silent) { - // Don't log empty cells - const stripped = this.extractExecutableLines(cell.data.source); - if (stripped.length > 0 && stripped.find(s => s.trim().length > 0)) { - // When the user adds new code, we know the execution count is increasing - this.executionCount += 1; - - // Skip hash on unknown file though - if (cell.file !== Identifiers.EmptyFileName) { - await this.addCellHash(cell, this.executionCount); - } - } - } - } catch (exc) { - // Don't let exceptions in a preExecute mess up normal operation - traceError(exc); - } - } - - public async postExecute(_cell: ICell, _silent: boolean): Promise { - noop(); - } - - private onChangedDocument(e: TextDocumentChangeEvent) { - // See if the document is in our list of docs to watch - const perFile = this.hashes.get(e.document.fileName); - if (perFile) { - // Apply the content changes to the file's cells. - const docText = e.document.getText(); - e.contentChanges.forEach(c => { - this.handleContentChange(docText, c, perFile); - }); - } - } - - private extractExecutableLines(code: nbformat.MultilineString): string[] { - const cellMatcher = new CellMatcher(this.configService.getSettings().datascience); - const lines = splitMultilineString(code); - // Only strip this off the first line. Otherwise we want the markers in the code. - if (lines.length > 0 && (cellMatcher.isCode(lines[0]) || cellMatcher.isMarkdown(lines[0]))) { - return lines.slice(1); - } - return lines; - } - - private handleContentChange(docText: string, c: TextDocumentContentChangeEvent, hashes: IRangedCellHash[]) { - // First compute the number of lines that changed - const lineDiff = c.range.start.line - c.range.end.line + c.text.split('\n').length - 1; - const offsetDiff = c.text.length - c.rangeLength; - - // Compute the inclusive offset that is changed by the cell. - const endChangedOffset = c.rangeLength <= 0 ? c.rangeOffset : c.rangeOffset + c.rangeLength - 1; - - hashes.forEach(h => { - // See how this existing cell compares to the change - if (h.endOffset < c.rangeOffset) { - // No change. This cell is entirely before the change - } else if (h.startOffset > endChangedOffset) { - // This cell is after the text that got replaced. Adjust its start/end lines - h.line += lineDiff; - h.endLine += lineDiff; - h.startOffset += offsetDiff; - h.endOffset += offsetDiff; - } else if (h.startOffset === endChangedOffset) { - // Cell intersects but exactly, might be a replacement or an insertion - if (h.deleted || c.rangeLength > 0 || lineDiff === 0) { - // Replacement - h.deleted = docText.substr(h.startOffset, h.endOffset - h.startOffset) !== h.realCode; - } else { - // Insertion - h.line += lineDiff; - h.endLine += lineDiff; - h.startOffset += offsetDiff; - h.endOffset += offsetDiff; - } - } else { - // Intersection, delete if necessary - h.deleted = docText.substr(h.startOffset, h.endOffset - h.startOffset) !== h.realCode; - } - }); - } - - private async addCellHash(cell: ICell, expectedCount: number): Promise { + public async addCellHash(cell: ICell, expectedCount: number): Promise { // Find the text document that matches. We need more information than // the add code gives us const doc = this.documentManager.textDocuments.find(d => this.fileSystem.arePathsSame(d.fileName, cell.file)); @@ -272,6 +186,65 @@ export class CellHashProvider implements ICellHashProvider, IInteractiveWindowLi } } + public extractExecutableLines(code: nbformat.MultilineString): string[] { + const cellMatcher = new CellMatcher(this.configService.getSettings().datascience); + const lines = splitMultilineString(code); + // Only strip this off the first line. Otherwise we want the markers in the code. + if (lines.length > 0 && (cellMatcher.isCode(lines[0]) || cellMatcher.isMarkdown(lines[0]))) { + return lines.slice(1); + } + return lines; + } + + private onChangedDocument(e: TextDocumentChangeEvent) { + // See if the document is in our list of docs to watch + const perFile = this.hashes.get(e.document.fileName); + if (perFile) { + // Apply the content changes to the file's cells. + const docText = e.document.getText(); + e.contentChanges.forEach(c => { + this.handleContentChange(docText, c, perFile); + }); + } + } + + private handleContentChange(docText: string, c: TextDocumentContentChangeEvent, hashes: IRangedCellHash[]) { + // First compute the number of lines that changed + const lineDiff = c.range.start.line - c.range.end.line + c.text.split('\n').length - 1; + const offsetDiff = c.text.length - c.rangeLength; + + // Compute the inclusive offset that is changed by the cell. + const endChangedOffset = c.rangeLength <= 0 ? c.rangeOffset : c.rangeOffset + c.rangeLength - 1; + + hashes.forEach(h => { + // See how this existing cell compares to the change + if (h.endOffset < c.rangeOffset) { + // No change. This cell is entirely before the change + } else if (h.startOffset > endChangedOffset) { + // This cell is after the text that got replaced. Adjust its start/end lines + h.line += lineDiff; + h.endLine += lineDiff; + h.startOffset += offsetDiff; + h.endOffset += offsetDiff; + } else if (h.startOffset === endChangedOffset) { + // Cell intersects but exactly, might be a replacement or an insertion + if (h.deleted || c.rangeLength > 0 || lineDiff === 0) { + // Replacement + h.deleted = docText.substr(h.startOffset, h.endOffset - h.startOffset) !== h.realCode; + } else { + // Insertion + h.line += lineDiff; + h.endLine += lineDiff; + h.startOffset += offsetDiff; + h.endOffset += offsetDiff; + } + } else { + // Intersection, delete if necessary + h.deleted = docText.substr(h.startOffset, h.endOffset - h.startOffset) !== h.realCode; + } + }); + } + private adjustRuntimeForDebugging(cell: ICell, source: string[], _cellStartOffset: number, _cellEndOffset: number): number { if (this.debugService.activeDebugSession && this.configService.getSettings().datascience.stopOnFirstLineWhileDebugging) { // Inject the breakpoint line diff --git a/src/client/datascience/editor-integration/codeLensFactory.ts b/src/client/datascience/editor-integration/codeLensFactory.ts index 1c3ed239dfb0..17723ef9d919 100644 --- a/src/client/datascience/editor-integration/codeLensFactory.ts +++ b/src/client/datascience/editor-integration/codeLensFactory.ts @@ -2,7 +2,7 @@ // Licensed under the MIT License. 'use strict'; import { inject, injectable } from 'inversify'; -import { CodeLens, Command, Event, EventEmitter, Range, TextDocument } from 'vscode'; +import { CodeLens, Command, Event, EventEmitter, Range, TextDocument, Uri } from 'vscode'; import { traceWarning } from '../../common/logger'; import { IFileSystem } from '../../common/platform/types'; @@ -11,8 +11,8 @@ import * as localize from '../../common/utils/localize'; import { noop } from '../../common/utils/misc'; import { generateCellRangesFromDocument } from '../cellFactory'; import { CodeLensCommands, Commands } from '../constants'; -import { InteractiveWindowMessages } from '../interactive-common/interactiveWindowTypes'; -import { ICell, ICellHashProvider, ICodeLensFactory, IFileHashes, IInteractiveWindowListener } from '../types'; +import { IInteractiveWindowMapping, InteractiveWindowMessages } from '../interactive-common/interactiveWindowTypes'; +import { ICell, ICellHashProvider, ICodeLensFactory, IFileHashes, IInteractiveWindowListener, IInteractiveWindowProvider, IJupyterExecution, INotebook } from '../types'; @injectable() export class CodeLensFactory implements ICodeLensFactory, IInteractiveWindowListener { @@ -20,14 +20,14 @@ export class CodeLensFactory implements ICodeLensFactory, IInteractiveWindowList // tslint:disable-next-line: no-any private postEmitter: EventEmitter<{ message: string; payload: any }> = new EventEmitter<{ message: string; payload: any }>(); private cellExecutionCounts: Map = new Map(); + private hashProvider: ICellHashProvider | undefined; constructor( @inject(IConfigurationService) private configService: IConfigurationService, - @inject(ICellHashProvider) private hashProvider: ICellHashProvider, + @inject(IInteractiveWindowProvider) private interactiveWindowProvider: IInteractiveWindowProvider, + @inject(IJupyterExecution) private jupyterExecution: IJupyterExecution, @inject(IFileSystem) private fileSystem: IFileSystem - ) { - hashProvider.updated(this.hashesUpdated.bind(this)); - } + ) { } public dispose(): void { noop(); @@ -41,6 +41,10 @@ export class CodeLensFactory implements ICodeLensFactory, IInteractiveWindowList // tslint:disable-next-line: no-any public onMessage(message: string, payload?: any) { switch (message) { + case InteractiveWindowMessages.NotebookExecutionActivated: + this.handleMessage(message, payload, this.doInitCellHashProvider); + break; + case InteractiveWindowMessages.FinishCell: const cell = payload as ICell; if (cell && cell.data && cell.data.execution_count) { @@ -65,7 +69,7 @@ export class CodeLensFactory implements ICodeLensFactory, IInteractiveWindowList public createCodeLenses(document: TextDocument): CodeLens[] { const ranges = generateCellRangesFromDocument(document, this.configService.getSettings().datascience); const commands = this.enumerateCommands(); - const hashes = this.configService.getSettings().datascience.addGotoCodeLenses ? this.hashProvider.getHashes() : []; + const hashes = this.configService.getSettings().datascience.addGotoCodeLenses && this.hashProvider ? this.hashProvider.getHashes() : []; const codeLenses: CodeLens[] = []; let firstCell = true; @@ -83,6 +87,40 @@ export class CodeLensFactory implements ICodeLensFactory, IInteractiveWindowList return codeLenses; } + // tslint:disable:no-any + private handleMessage(_message: T, payload: any, handler: (args: M[T]) => void) { + const args = payload as M[T]; + handler.bind(this)(args); + } + + private doInitCellHashProvider(payload: string): void { + this.initCellHashProvider(payload).ignoreErrors(); + } + + private async initCellHashProvider(notebookUri: string) { + const nbUri: Uri = Uri.parse(notebookUri); + if (!nbUri) { + return; + } + + // First get the active server + const activeServer = await this.jupyterExecution.getServer(await this.interactiveWindowProvider.getNotebookOptions()); + + let nb: INotebook | undefined; + // If that works, see if there's a matching notebook running + if (activeServer) { + nb = await activeServer.getNotebook(nbUri); + + // If we have an executing notebook, get its gather execution service. + if (nb) { + this.hashProvider = nb.getCellHashProvider(); + if (this.hashProvider) { + this.hashProvider.updated(this.hashesUpdated.bind(this)); + } + } + } + } + private enumerateCommands(): string[] { let fullCommandList: string[]; // Add our non-debug commands diff --git a/src/client/datascience/gather/gatherListener.ts b/src/client/datascience/gather/gatherListener.ts index ed34d796d28c..cd9c42a8a325 100644 --- a/src/client/datascience/gather/gatherListener.ts +++ b/src/client/datascience/gather/gatherListener.ts @@ -37,8 +37,7 @@ export class GatherListener implements IInteractiveWindowListener { @inject(IInteractiveWindowProvider) private interactiveWindowProvider: IInteractiveWindowProvider, @inject(IDocumentManager) private documentManager: IDocumentManager, @inject(IFileSystem) private fileSystem: IFileSystem - ) { - } + ) { } public dispose() { noop(); @@ -106,7 +105,7 @@ export class GatherListener implements IInteractiveWindowListener { } private gatherCodeInternal = async (cell: ICell) => { - const slicedProgram = (this.gatherService) ? this.gatherService.gatherCode(cell) : ''; + const slicedProgram = this.gatherService ? this.gatherService.gatherCode(cell) : ''; if (this.configService.getSettings().datascience.gatherToScript) { await this.showFile(slicedProgram, cell.file); diff --git a/src/client/datascience/gather/gatherLogger.ts b/src/client/datascience/gather/gatherLogger.ts index 5c43347226a7..d10f8261b787 100644 --- a/src/client/datascience/gather/gatherLogger.ts +++ b/src/client/datascience/gather/gatherLogger.ts @@ -5,16 +5,11 @@ import { IConfigurationService } from '../../common/types'; import { noop } from '../../common/utils/misc'; import { CellMatcher } from '../cellMatcher'; import { concatMultilineStringInput } from '../common'; -import { ICell as IVscCell, IGatherExecution, IGatherLogger, INotebookExecutionLogger } from '../types'; +import { ICell as IVscCell, IGatherExecution, IGatherLogger } from '../types'; @injectable() -export class GatherLogger implements INotebookExecutionLogger, IGatherLogger { - - constructor( - @inject(IGatherExecution) private gather: IGatherExecution, - @inject(IConfigurationService) private configService: IConfigurationService - ) { - } +export class GatherLogger implements IGatherLogger { + constructor(@inject(IGatherExecution) private gather: IGatherExecution, @inject(IConfigurationService) private configService: IConfigurationService) { } public async preExecute(_vscCell: IVscCell, _silent: boolean): Promise { // This function is just implemented here for compliance with the INotebookExecutionLogger interface diff --git a/src/client/datascience/jupyter/jupyterNotebook.ts b/src/client/datascience/jupyter/jupyterNotebook.ts index c86e1dbf3f42..cb3adbe8000b 100644 --- a/src/client/datascience/jupyter/jupyterNotebook.ts +++ b/src/client/datascience/jupyter/jupyterNotebook.ts @@ -29,6 +29,8 @@ import { CodeSnippits, Identifiers, Telemetry } from '../constants'; import { CellState, ICell, + ICellHashLogger, + ICellHashProvider, IGatherExecution, IGatherLogger, IJupyterKernelSpec, @@ -568,10 +570,19 @@ export class JupyterNotebookBase implements INotebook { } public getGatherService(): IGatherExecution | undefined { - const gatherLogger: INotebookExecutionLogger | undefined = this._loggers.find((logger: INotebookExecutionLogger) => { return (logger).service !== undefined; }); + const gatherLogger: INotebookExecutionLogger | undefined = this._loggers.find((logger: INotebookExecutionLogger) => { + return (logger).service !== undefined; + }); return gatherLogger ? (gatherLogger).service() : undefined; } + public getCellHashProvider(): ICellHashProvider | undefined { + const cellHashLogger: INotebookExecutionLogger | undefined = this._loggers.find((logger: INotebookExecutionLogger) => { + return (logger).getProvider !== undefined; + }); + return cellHashLogger ? (cellHashLogger).getProvider() : undefined; + } + private async initializeMatplotlib(cancelToken?: CancellationToken): Promise { const settings = this.configService.getSettings().datascience; if (settings && settings.themeMatplotlibPlots) { @@ -675,15 +686,15 @@ export class JupyterNotebookBase implements INotebook { const cellMatcher = new CellMatcher(this.configService.getSettings().datascience); return this.session ? this.session.requestExecute( - { - // Remove the cell marker if we have one. - code: cellMatcher.stripFirstMarker(code), - stop_on_error: false, - allow_stdin: true, // Allow when silent too in case runStartupCommands asks for a password - store_history: !silent // Silent actually means don't output anything. Store_history is what affects execution_count - }, - true - ) + { + // Remove the cell marker if we have one. + code: cellMatcher.stripFirstMarker(code), + stop_on_error: false, + allow_stdin: true, // Allow when silent too in case runStartupCommands asks for a password + store_history: !silent // Silent actually means don't output anything. Store_history is what affects execution_count + }, + true + ) : undefined; } catch (exc) { // Any errors generating a request should just be logged. User can't do anything about it. diff --git a/src/client/datascience/jupyter/liveshare/guestJupyterNotebook.ts b/src/client/datascience/jupyter/liveshare/guestJupyterNotebook.ts index 95a3d4e4c865..8be7b3b67bc0 100644 --- a/src/client/datascience/jupyter/liveshare/guestJupyterNotebook.ts +++ b/src/client/datascience/jupyter/liveshare/guestJupyterNotebook.ts @@ -16,7 +16,7 @@ import * as localize from '../../../common/utils/localize'; import { noop } from '../../../common/utils/misc'; import { PythonInterpreter } from '../../../interpreter/contracts'; import { LiveShare, LiveShareCommands } from '../../constants'; -import { ICell, IGatherExecution, IJupyterKernelSpec, INotebook, INotebookCompletion, INotebookServer, InterruptResult } from '../../types'; +import { ICell, ICellHashProvider, IGatherExecution, IJupyterKernelSpec, INotebook, INotebookCompletion, INotebookServer, InterruptResult } from '../../types'; import { LiveKernelModel } from '../kernels/types'; import { LiveShareParticipantDefault, LiveShareParticipantGuest } from './liveShareParticipantMixin'; import { ResponseQueue } from './responseQueue'; @@ -206,6 +206,10 @@ export class GuestJupyterNotebook extends LiveShareParticipantGuest(LiveSharePar return; } + public getCellHashProvider(): ICellHashProvider | undefined { + return; + } + public setKernelSpec(_spec: IJupyterKernelSpec | LiveKernelModel, _timeout: number): Promise { return Promise.resolve(); } diff --git a/src/client/datascience/serviceRegistry.ts b/src/client/datascience/serviceRegistry.ts index 68be2973c00f..335b99df8653 100644 --- a/src/client/datascience/serviceRegistry.ts +++ b/src/client/datascience/serviceRegistry.ts @@ -12,6 +12,7 @@ import { DataViewer } from './data-viewing/dataViewer'; import { DataViewerProvider } from './data-viewing/dataViewerProvider'; import { DataScience } from './datascience'; import { DebugLocationTrackerFactory } from './debugLocationTrackerFactory'; +import { CellHashLogger } from './editor-integration/cellhashLogger'; import { CellHashProvider } from './editor-integration/cellhashprovider'; import { CodeLensFactory } from './editor-integration/codeLensFactory'; import { DataScienceCodeLensProvider } from './editor-integration/codelensprovider'; @@ -20,6 +21,7 @@ import { Decorator } from './editor-integration/decorator'; import { DataScienceErrorHandler } from './errorHandler/errorHandler'; import { GatherExecution } from './gather/gather'; import { GatherListener } from './gather/gatherListener'; +import { GatherLogger } from './gather/gatherLogger'; import { DebugListener } from './interactive-common/debugListener'; import { IntellisenseProvider } from './interactive-common/intellisense/intellisenseProvider'; import { LinkProvider } from './interactive-common/linkProvider'; @@ -58,6 +60,7 @@ import { StatusProvider } from './statusProvider'; import { ThemeFinder } from './themeFinder'; import { ICellHashListener, + ICellHashLogger, ICellHashProvider, ICodeCssGenerator, ICodeLensFactory, @@ -91,7 +94,6 @@ import { IStatusProvider, IThemeFinder } from './types'; -import { GatherLogger } from './gather/gatherLogger'; export function registerTypes(serviceManager: IServiceManager) { serviceManager.addSingleton(IDataScienceCodeLensProvider, DataScienceCodeLensProvider); @@ -125,12 +127,16 @@ export function registerTypes(serviceManager: IServiceManager) { serviceManager.addSingleton(IJupyterDebugger, JupyterDebugger); serviceManager.add(IDataScienceErrorHandler, DataScienceErrorHandler); serviceManager.addSingleton(ICodeLensFactory, CodeLensFactory); + serviceManager.add(ICellHashProvider, CellHashProvider); - serviceManager.add(IGatherLogger, GatherLogger); - serviceManager.add(IGatherExecution, GatherExecution); + serviceManager.add(ICellHashLogger, CellHashLogger); + serviceManager.addBinding(ICellHashLogger, INotebookExecutionLogger); serviceManager.addBinding(ICellHashProvider, IInteractiveWindowListener); - serviceManager.addBinding(ICellHashProvider, INotebookExecutionLogger); + + serviceManager.add(IGatherExecution, GatherExecution); + serviceManager.add(IGatherLogger, GatherLogger); serviceManager.addBinding(IGatherLogger, INotebookExecutionLogger); + serviceManager.addBinding(IJupyterDebugger, ICellHashListener); serviceManager.addSingleton(INotebookEditorProvider, NativeEditorProvider); serviceManager.add(INotebookEditor, NativeEditor); diff --git a/src/client/datascience/types.ts b/src/client/datascience/types.ts index 7baf7dbe2b01..511c9ffff36f 100644 --- a/src/client/datascience/types.ts +++ b/src/client/datascience/types.ts @@ -104,6 +104,7 @@ export interface INotebook extends IAsyncDisposable { setKernelSpec(spec: IJupyterKernelSpec | LiveKernelModel, timeoutMS: number): Promise; setInterpreter(interpeter: PythonInterpreter): void; getGatherService(): IGatherExecution | undefined; + getCellHashProvider(): ICellHashProvider | undefined; } export interface INotebookServerOptions { @@ -589,6 +590,11 @@ export interface ICellHashProvider { getHashes(): IFileHashes[]; } +export const ICellHashLogger = Symbol('ICellHashLogger'); +export interface ICellHashLogger extends INotebookExecutionLogger { + getProvider(): ICellHashProvider; +} + export interface IDebugLocation { fileName: string; lineNumber: number; diff --git a/src/test/datascience/editor-integration/cellhashprovider.unit.test.ts b/src/test/datascience/editor-integration/cellhashprovider.unit.test.ts index 6b80008bc6d7..2f444b67a1d6 100644 --- a/src/test/datascience/editor-integration/cellhashprovider.unit.test.ts +++ b/src/test/datascience/editor-integration/cellhashprovider.unit.test.ts @@ -8,6 +8,7 @@ import { Position, Range, Uri } from 'vscode'; import { IDebugService } from '../../../client/common/application/types'; import { IFileSystem } from '../../../client/common/platform/types'; import { IConfigurationService, IDataScienceSettings, IPythonSettings } from '../../../client/common/types'; +import { CellHashLogger } from '../../../client/datascience/editor-integration/cellhashLogger'; import { CellHashProvider } from '../../../client/datascience/editor-integration/cellhashprovider'; import { InteractiveWindowMessages, SysInfoReason } from '../../../client/datascience/interactive-common/interactiveWindowTypes'; import { CellState, ICell, ICellHashListener, IFileHashes } from '../../../client/datascience/types'; @@ -24,6 +25,7 @@ class HashListener implements ICellHashListener { // tslint:disable-next-line: max-func-body-length suite('CellHashProvider Unit Tests', () => { let hashProvider: CellHashProvider; + let hashLogger: CellHashLogger; let documentManager: MockDocumentManager; let configurationService: TypeMoq.IMock; let dataScienceSettings: TypeMoq.IMock; @@ -44,6 +46,7 @@ suite('CellHashProvider Unit Tests', () => { fileSystem.setup(d => d.arePathsSame(TypeMoq.It.isAnyString(), TypeMoq.It.isAnyString())).returns(() => true); documentManager = new MockDocumentManager(); hashProvider = new CellHashProvider(documentManager, configurationService.object, debugService.object, fileSystem.object, [hashListener]); + hashLogger = new CellHashLogger(hashProvider); }); function addSingleChange(file: string, range: Range, newText: string) { @@ -64,7 +67,7 @@ suite('CellHashProvider Unit Tests', () => { id: '1', state: CellState.init }; - return hashProvider.preExecute(cell, false); + return hashLogger.preExecute(cell, false); } test('Add a cell and edit it', async () => { diff --git a/src/test/datascience/editor-integration/codewatcher.unit.test.ts b/src/test/datascience/editor-integration/codewatcher.unit.test.ts index 6a6b0e4f41e9..938d59e6512d 100644 --- a/src/test/datascience/editor-integration/codewatcher.unit.test.ts +++ b/src/test/datascience/editor-integration/codewatcher.unit.test.ts @@ -21,7 +21,8 @@ import { IDataScienceErrorHandler, IDebugLocationTracker, IInteractiveWindow, - IInteractiveWindowProvider + IInteractiveWindowProvider, + IJupyterExecution } from '../../../client/datascience/types'; import { IServiceContainer } from '../../../client/ioc/types'; import { ICodeExecutionHelper } from '../../../client/terminals/types'; @@ -33,6 +34,7 @@ import { createDocument } from './helpers'; suite('DataScience Code Watcher Unit Tests', () => { let codeWatcher: CodeWatcher; let interactiveWindowProvider: TypeMoq.IMock; + let jupyterExecution: TypeMoq.IMock; let activeInteractiveWindow: TypeMoq.IMock; let documentManager: TypeMoq.IMock; let commandManager: TypeMoq.IMock; @@ -45,7 +47,6 @@ suite('DataScience Code Watcher Unit Tests', () => { let tokenSource: CancellationTokenSource; let debugService: TypeMoq.IMock; let debugLocationTracker: TypeMoq.IMock; - let cellHashProvider: TypeMoq.IMock; const contexts: Map = new Map(); const pythonSettings = new (class extends PythonSettings { public fireChangeEvent() { @@ -57,6 +58,7 @@ suite('DataScience Code Watcher Unit Tests', () => { setup(() => { tokenSource = new CancellationTokenSource(); interactiveWindowProvider = TypeMoq.Mock.ofType(); + jupyterExecution = TypeMoq.Mock.ofType(); activeInteractiveWindow = createTypeMoq('history'); documentManager = TypeMoq.Mock.ofType(); textEditor = TypeMoq.Mock.ofType(); @@ -66,7 +68,6 @@ suite('DataScience Code Watcher Unit Tests', () => { helper = TypeMoq.Mock.ofType(); commandManager = TypeMoq.Mock.ofType(); debugService = TypeMoq.Mock.ofType(); - cellHashProvider = TypeMoq.Mock.ofType(); // Setup default settings pythonSettings.datascience = { @@ -103,7 +104,7 @@ suite('DataScience Code Watcher Unit Tests', () => { // Setup the file system fileSystem.setup(f => f.arePathsSame(TypeMoq.It.isAnyString(), TypeMoq.It.isAnyString())).returns(() => true); - const codeLensFactory = new CodeLensFactory(configService.object, cellHashProvider.object, fileSystem.object); + const codeLensFactory = new CodeLensFactory(configService.object, interactiveWindowProvider.object, jupyterExecution.object, fileSystem.object); serviceContainer .setup(c => c.get(TypeMoq.It.isValue(ICodeWatcher))) .returns( @@ -140,7 +141,7 @@ suite('DataScience Code Watcher Unit Tests', () => { return Promise.resolve(); }); - const codeLens = new CodeLensFactory(configService.object, cellHashProvider.object, fileSystem.object); + const codeLens = new CodeLensFactory(configService.object, interactiveWindowProvider.object, jupyterExecution.object, fileSystem.object); codeWatcher = new CodeWatcher( interactiveWindowProvider.object, diff --git a/src/test/datascience/execution.unit.test.ts b/src/test/datascience/execution.unit.test.ts index 930f5aaa07cf..a4a750fb71ec 100644 --- a/src/test/datascience/execution.unit.test.ts +++ b/src/test/datascience/execution.unit.test.ts @@ -50,6 +50,7 @@ import { NotebookStarter } from '../../client/datascience/jupyter/notebookStarte import { LiveShareApi } from '../../client/datascience/liveshare/liveshare'; import { ICell, + ICellHashProvider, IConnection, IGatherExecution, IJupyterKernelSpec, @@ -155,6 +156,10 @@ class MockJupyterNotebook implements INotebook { return; } + public getCellHashProvider(): ICellHashProvider | undefined { + return; + } + public get onSessionStatusChanged(): Event { if (!this.onStatusChangedEvent) { this.onStatusChangedEvent = new EventEmitter(); From c69d0c2d67dcaf498736dce7d201a111a147ab10 Mon Sep 17 00:00:00 2001 From: Jim Griesmer Date: Fri, 17 Jan 2020 17:32:54 -0800 Subject: [PATCH 10/51] Rename gather* to match existing classes and interfaces --- src/client/datascience/gather/gather.ts | 4 ++-- .../datascience/gather/gatherListener.ts | 21 ++++++------------- src/client/datascience/gather/gatherLogger.ts | 4 ++-- .../datascience/jupyter/jupyterNotebook.ts | 4 ++-- .../jupyter/liveshare/guestJupyterNotebook.ts | 4 ++-- src/client/datascience/serviceRegistry.ts | 4 ++-- src/client/datascience/types.ts | 8 +++---- .../datascience/dataScienceIocContainer.ts | 4 ++-- .../codewatcher.unit.test.ts | 1 - src/test/datascience/execution.unit.test.ts | 4 ++-- 10 files changed, 24 insertions(+), 34 deletions(-) diff --git a/src/client/datascience/gather/gather.ts b/src/client/datascience/gather/gather.ts index 5228911729ba..2f23930a7ff6 100644 --- a/src/client/datascience/gather/gather.ts +++ b/src/client/datascience/gather/gather.ts @@ -9,13 +9,13 @@ import * as localize from '../../common/utils/localize'; // tslint:disable-next-line: no-duplicate-imports import { Common } from '../../common/utils/localize'; import { Identifiers } from '../constants'; -import { CellState, ICell as IVscCell, IGatherExecution } from '../types'; +import { CellState, ICell as IVscCell, IGatherProvider } from '../types'; /** * An adapter class to wrap the code gathering functionality from [microsoft/python-program-analysis](https://www.npmjs.com/package/@msrvida/python-program-analysis). */ @injectable() -export class GatherExecution implements IGatherExecution { +export class GatherExecution implements IGatherProvider { private _executionSlicer: ExecutionLogSlicer; private dataflowAnalyzer: DataflowAnalyzer; private _enabled: boolean; diff --git a/src/client/datascience/gather/gatherListener.ts b/src/client/datascience/gather/gatherListener.ts index aa1a39a2912b..ae87302ff6f2 100644 --- a/src/client/datascience/gather/gatherListener.ts +++ b/src/client/datascience/gather/gatherListener.ts @@ -11,23 +11,14 @@ import { noop } from '../../common/utils/misc'; import { generateCellsFromString } from '../cellFactory'; import { Identifiers } from '../constants'; import { IInteractiveWindowMapping, InteractiveWindowMessages } from '../interactive-common/interactiveWindowTypes'; -import { - ICell, - IGatherExecution, - IInteractiveWindowListener, - IInteractiveWindowProvider, - IJupyterExecution, - INotebook, - INotebookEditorProvider, - INotebookExporter -} from '../types'; +import { ICell, IGatherProvider, IInteractiveWindowListener, IInteractiveWindowProvider, IJupyterExecution, INotebook, INotebookEditorProvider, INotebookExporter } from '../types'; @injectable() export class GatherListener implements IInteractiveWindowListener { // tslint:disable-next-line: no-any private postEmitter: EventEmitter<{ message: string; payload: any }> = new EventEmitter<{ message: string; payload: any }>(); private notebookUri: Uri | undefined; - private gatherService: IGatherExecution | undefined; + private gatherProvider: IGatherProvider | undefined; constructor( @inject(IApplicationShell) private applicationShell: IApplicationShell, @@ -61,8 +52,8 @@ export class GatherListener implements IInteractiveWindowListener { break; case InteractiveWindowMessages.RestartKernel: - if (this.gatherService) { - this.gatherService.resetLog(); + if (this.gatherProvider) { + this.gatherProvider.resetLog(); } break; @@ -100,13 +91,13 @@ export class GatherListener implements IInteractiveWindowListener { // If we have an executing notebook, get its gather execution service. if (nb) { - this.gatherService = nb.getGatherService(); + this.gatherProvider = nb.getgatherProvider(); } } } private gatherCodeInternal = async (cell: ICell) => { - const slicedProgram = this.gatherService ? this.gatherService.gatherCode(cell) : ''; + const slicedProgram = this.gatherProvider ? this.gatherProvider.gatherCode(cell) : ''; if (this.configService.getSettings().datascience.gatherToScript) { await this.showFile(slicedProgram, cell.file); diff --git a/src/client/datascience/gather/gatherLogger.ts b/src/client/datascience/gather/gatherLogger.ts index 66bacef33f8f..d7ba8a0511f3 100644 --- a/src/client/datascience/gather/gatherLogger.ts +++ b/src/client/datascience/gather/gatherLogger.ts @@ -5,11 +5,11 @@ import { concatMultilineStringInput } from '../../../datascience-ui/common'; import { IConfigurationService } from '../../common/types'; import { noop } from '../../common/utils/misc'; import { CellMatcher } from '../cellMatcher'; -import { ICell as IVscCell, IGatherExecution, IGatherLogger } from '../types'; +import { ICell as IVscCell, IGatherLogger, IGatherProvider } from '../types'; @injectable() export class GatherLogger implements IGatherLogger { - constructor(@inject(IGatherExecution) private gather: IGatherExecution, @inject(IConfigurationService) private configService: IConfigurationService) { } + constructor(@inject(IGatherProvider) private gather: IGatherProvider, @inject(IConfigurationService) private configService: IConfigurationService) { } public async preExecute(_vscCell: IVscCell, _silent: boolean): Promise { // This function is just implemented here for compliance with the INotebookExecutionLogger interface diff --git a/src/client/datascience/jupyter/jupyterNotebook.ts b/src/client/datascience/jupyter/jupyterNotebook.ts index fc7e01b01852..ca0331483e87 100644 --- a/src/client/datascience/jupyter/jupyterNotebook.ts +++ b/src/client/datascience/jupyter/jupyterNotebook.ts @@ -30,8 +30,8 @@ import { ICell, ICellHashLogger, ICellHashProvider, - IGatherExecution, IGatherLogger, + IGatherProvider, IJupyterKernelSpec, IJupyterSession, INotebook, @@ -569,7 +569,7 @@ export class JupyterNotebookBase implements INotebook { this.kernelChanged.fire(spec); } - public getGatherService(): IGatherExecution | undefined { + public getgatherProvider(): IGatherProvider | undefined { const gatherLogger: INotebookExecutionLogger | undefined = this._loggers.find((logger: INotebookExecutionLogger) => { return (logger).service !== undefined; }); diff --git a/src/client/datascience/jupyter/liveshare/guestJupyterNotebook.ts b/src/client/datascience/jupyter/liveshare/guestJupyterNotebook.ts index 8be7b3b67bc0..f79545dd7aca 100644 --- a/src/client/datascience/jupyter/liveshare/guestJupyterNotebook.ts +++ b/src/client/datascience/jupyter/liveshare/guestJupyterNotebook.ts @@ -16,7 +16,7 @@ import * as localize from '../../../common/utils/localize'; import { noop } from '../../../common/utils/misc'; import { PythonInterpreter } from '../../../interpreter/contracts'; import { LiveShare, LiveShareCommands } from '../../constants'; -import { ICell, ICellHashProvider, IGatherExecution, IJupyterKernelSpec, INotebook, INotebookCompletion, INotebookServer, InterruptResult } from '../../types'; +import { ICell, ICellHashProvider, IGatherProvider, IJupyterKernelSpec, INotebook, INotebookCompletion, INotebookServer, InterruptResult } from '../../types'; import { LiveKernelModel } from '../kernels/types'; import { LiveShareParticipantDefault, LiveShareParticipantGuest } from './liveShareParticipantMixin'; import { ResponseQueue } from './responseQueue'; @@ -202,7 +202,7 @@ export class GuestJupyterNotebook extends LiveShareParticipantGuest(LiveSharePar return; } - public getGatherService(): IGatherExecution | undefined { + public getgatherProvider(): IGatherProvider | undefined { return; } diff --git a/src/client/datascience/serviceRegistry.ts b/src/client/datascience/serviceRegistry.ts index 6251ce0bd326..f219cf36ecc4 100644 --- a/src/client/datascience/serviceRegistry.ts +++ b/src/client/datascience/serviceRegistry.ts @@ -79,8 +79,8 @@ import { IDataViewer, IDataViewerProvider, IDebugLocationTracker, - IGatherExecution, IGatherLogger, + IGatherProvider, IInteractiveWindow, IInteractiveWindowListener, IInteractiveWindowProvider, @@ -142,7 +142,7 @@ export function registerTypes(serviceManager: IServiceManager) { serviceManager.addBinding(ICellHashLogger, INotebookExecutionLogger); serviceManager.addBinding(ICellHashProvider, IInteractiveWindowListener); - serviceManager.add(IGatherExecution, GatherExecution); + serviceManager.add(IGatherProvider, GatherExecution); serviceManager.add(IGatherLogger, GatherLogger); serviceManager.addBinding(IGatherLogger, INotebookExecutionLogger); diff --git a/src/client/datascience/types.ts b/src/client/datascience/types.ts index 8499731a69d0..9ab9e2451889 100644 --- a/src/client/datascience/types.ts +++ b/src/client/datascience/types.ts @@ -109,7 +109,7 @@ export interface INotebook extends IAsyncDisposable { getKernelSpec(): IJupyterKernelSpec | LiveKernelModel | undefined; setKernelSpec(spec: IJupyterKernelSpec | LiveKernelModel, timeoutMS: number): Promise; setInterpreter(interpeter: PythonInterpreter): void; - getGatherService(): IGatherExecution | undefined; + getgatherProvider(): IGatherProvider | undefined; getCellHashProvider(): ICellHashProvider | undefined; } @@ -129,8 +129,8 @@ export interface INotebookExecutionLogger { postExecute(cell: ICell, silent: boolean): Promise; } -export const IGatherExecution = Symbol('IGatherExecution'); -export interface IGatherExecution { +export const IGatherProvider = Symbol('IGatherProvider'); +export interface IGatherProvider { enabled: boolean; logExecution(vscCell: ICell): void; gatherCode(vscCell: ICell): string; @@ -139,7 +139,7 @@ export interface IGatherExecution { export const IGatherLogger = Symbol('IGatherLogger'); export interface IGatherLogger extends INotebookExecutionLogger { - service(): IGatherExecution; + service(): IGatherProvider; } export const IJupyterExecution = Symbol('IJupyterExecution'); diff --git a/src/test/datascience/dataScienceIocContainer.ts b/src/test/datascience/dataScienceIocContainer.ts index d0f3ae6afbcc..fbee501e3375 100644 --- a/src/test/datascience/dataScienceIocContainer.ts +++ b/src/test/datascience/dataScienceIocContainer.ts @@ -192,7 +192,7 @@ import { IDataViewer, IDataViewerProvider, IDebugLocationTracker, - IGatherExecution, + IGatherProvider, IInteractiveWindow, IInteractiveWindowListener, IInteractiveWindowProvider, @@ -475,7 +475,7 @@ export class DataScienceIocContainer extends UnitTestIocContainer { this.serviceManager.add(IInteractiveWindowListener, GatherListener); this.serviceManager.addBinding(ICellHashProvider, INotebookExecutionLogger); this.serviceManager.addBinding(IJupyterDebugger, ICellHashListener); - this.serviceManager.add(IGatherExecution, GatherExecution); + this.serviceManager.add(IGatherProvider, GatherExecution); this.serviceManager.addSingleton(ICodeLensFactory, CodeLensFactory); this.serviceManager.addSingleton(IShellDetector, TerminalNameShellDetector); this.serviceManager.addSingleton(InterpeterHashProviderFactory, InterpeterHashProviderFactory); diff --git a/src/test/datascience/editor-integration/codewatcher.unit.test.ts b/src/test/datascience/editor-integration/codewatcher.unit.test.ts index 938d59e6512d..0d5a87a11b27 100644 --- a/src/test/datascience/editor-integration/codewatcher.unit.test.ts +++ b/src/test/datascience/editor-integration/codewatcher.unit.test.ts @@ -16,7 +16,6 @@ import { CodeLensFactory } from '../../../client/datascience/editor-integration/ import { DataScienceCodeLensProvider } from '../../../client/datascience/editor-integration/codelensprovider'; import { CodeWatcher } from '../../../client/datascience/editor-integration/codewatcher'; import { - ICellHashProvider, ICodeWatcher, IDataScienceErrorHandler, IDebugLocationTracker, diff --git a/src/test/datascience/execution.unit.test.ts b/src/test/datascience/execution.unit.test.ts index 6706c38daeb6..2f0cece8915a 100644 --- a/src/test/datascience/execution.unit.test.ts +++ b/src/test/datascience/execution.unit.test.ts @@ -51,7 +51,7 @@ import { ICell, ICellHashProvider, IConnection, - IGatherExecution, + IGatherProvider, IJupyterKernelSpec, IJupyterSubCommandExecutionService, INotebook, @@ -152,7 +152,7 @@ class MockJupyterNotebook implements INotebook { return Promise.resolve(); } - public getGatherService(): IGatherExecution | undefined { + public getgatherProvider(): IGatherProvider | undefined { return; } From cb7d0a75422e4d2c831e573cd78a0ae58af1ed59 Mon Sep 17 00:00:00 2001 From: Jim Griesmer Date: Tue, 21 Jan 2020 11:55:38 -0800 Subject: [PATCH 11/51] Update to latest python analysis --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 3092528a4cca..409285f6ea22 100644 --- a/package.json +++ b/package.json @@ -2788,7 +2788,7 @@ "@jupyterlab/coreutils": "^3.1.0", "@jupyterlab/services": "^4.2.0", "@koa/cors": "^3.0.0", - "@msrvida/python-program-analysis": "^0.4.12", + "@msrvida/python-program-analysis": "^0.4.20", "ansi-regex": "^4.1.0", "arch": "^2.1.0", "azure-storage": "^2.10.3", From 7a7c773cebfbb399ffc4467d1886d0b602bcdf0b Mon Sep 17 00:00:00 2001 From: Jim Griesmer Date: Tue, 21 Jan 2020 11:57:22 -0800 Subject: [PATCH 12/51] lock python analysis ver --- package-lock.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index 6a97f71e0848..a7772870df8d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1427,9 +1427,9 @@ } }, "@msrvida/python-program-analysis": { - "version": "0.4.12", - "resolved": "https://pkgs.dev.azure.com/msresearch/_packaging/MSR-Python-Analysis/npm/registry/@msrvida/python-program-analysis/-/python-program-analysis-0.4.12.tgz", - "integrity": "sha1-ic1eFXy+eM1t5xFxdxIDAfSYza0=" + "version": "0.4.20", + "resolved": "https://pkgs.dev.azure.com/msresearch/_packaging/MSR-Python-Analysis/npm/registry/@msrvida/python-program-analysis/-/python-program-analysis-0.4.20.tgz", + "integrity": "sha1-NF+FuDLhWe7U5KkL48LDx8P1He8=" }, "@nteract/markdown": { "version": "3.0.1", From fe2c87f405feee176016244dc83bb32b4975c87b Mon Sep 17 00:00:00 2001 From: Jim Griesmer Date: Thu, 23 Jan 2020 23:46:27 -0800 Subject: [PATCH 13/51] Fix MockJupyterNotebook --- src/test/datascience/mockJupyterNotebook.ts | 51 +++++++++++++-------- 1 file changed, 33 insertions(+), 18 deletions(-) diff --git a/src/test/datascience/mockJupyterNotebook.ts b/src/test/datascience/mockJupyterNotebook.ts index a9621def81ea..a3e355876a92 100644 --- a/src/test/datascience/mockJupyterNotebook.ts +++ b/src/test/datascience/mockJupyterNotebook.ts @@ -6,18 +6,22 @@ import { Observable } from 'rxjs/Observable'; import { CancellationToken, Event, EventEmitter, Uri } from 'vscode'; import { Identifiers } from '../../client/datascience/constants'; import { LiveKernelModel } from '../../client/datascience/jupyter/kernels/types'; -import { ICell, IJupyterKernelSpec, INotebook, INotebookCompletion, INotebookExecutionLogger, INotebookServer, InterruptResult } from '../../client/datascience/types'; +import { + ICell, + ICellHashProvider, + IGatherProvider, + IJupyterKernelSpec, + INotebook, + INotebookCompletion, + INotebookExecutionLogger, + INotebookServer, + InterruptResult +} from '../../client/datascience/types'; import { PythonInterpreter } from '../../client/interpreter/contracts'; import { ServerStatus } from '../../datascience-ui/interactive-common/mainState'; import { noop } from '../core'; export class MockJupyterNotebook implements INotebook { - public onKernelChanged: Event = new EventEmitter().event; - private onStatusChangedEvent: EventEmitter | undefined; - constructor(private owner: INotebookServer) { - noop(); - } - public get server(): INotebookServer { return this.owner; } @@ -26,6 +30,28 @@ export class MockJupyterNotebook implements INotebook { return Uri.parse(Identifiers.InteractiveWindowIdentity); } + public get onSessionStatusChanged(): Event { + if (!this.onStatusChangedEvent) { + this.onStatusChangedEvent = new EventEmitter(); + } + return this.onStatusChangedEvent.event; + } + + public get status(): ServerStatus { + return ServerStatus.Idle; + } + public onKernelChanged: Event = new EventEmitter().event; + private onStatusChangedEvent: EventEmitter | undefined; + constructor(private owner: INotebookServer) { + noop(); + } + public getgatherProvider(): IGatherProvider | undefined { + throw new Error('Method not implemented.'); + } + public getCellHashProvider(): ICellHashProvider | undefined { + throw new Error('Method not implemented.'); + } + public clear(_id: string): void { noop(); } @@ -94,15 +120,4 @@ export class MockJupyterNotebook implements INotebook { public setKernelSpec(_spec: IJupyterKernelSpec | LiveKernelModel, _timeout: number): Promise { return Promise.resolve(); } - - public get onSessionStatusChanged(): Event { - if (!this.onStatusChangedEvent) { - this.onStatusChangedEvent = new EventEmitter(); - } - return this.onStatusChangedEvent.event; - } - - public get status(): ServerStatus { - return ServerStatus.Idle; - } } From a65f35e1da4726d75b465a10946c5021ef09d8c5 Mon Sep 17 00:00:00 2001 From: Jim Griesmer Date: Thu, 23 Jan 2020 23:56:50 -0800 Subject: [PATCH 14/51] Fix casing of getgatherProvider --- src/client/datascience/gather/gatherListener.ts | 2 +- src/client/datascience/jupyter/jupyterNotebook.ts | 2 +- .../datascience/jupyter/liveshare/guestJupyterNotebook.ts | 2 +- src/client/datascience/types.ts | 2 +- src/test/datascience/mockJupyterNotebook.ts | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/client/datascience/gather/gatherListener.ts b/src/client/datascience/gather/gatherListener.ts index ae87302ff6f2..b671d6cf36af 100644 --- a/src/client/datascience/gather/gatherListener.ts +++ b/src/client/datascience/gather/gatherListener.ts @@ -91,7 +91,7 @@ export class GatherListener implements IInteractiveWindowListener { // If we have an executing notebook, get its gather execution service. if (nb) { - this.gatherProvider = nb.getgatherProvider(); + this.gatherProvider = nb.getGatherProvider(); } } } diff --git a/src/client/datascience/jupyter/jupyterNotebook.ts b/src/client/datascience/jupyter/jupyterNotebook.ts index 46fcedf42f26..91dd1e2a07c5 100644 --- a/src/client/datascience/jupyter/jupyterNotebook.ts +++ b/src/client/datascience/jupyter/jupyterNotebook.ts @@ -576,7 +576,7 @@ export class JupyterNotebookBase implements INotebook { this.kernelChanged.fire(spec); } - public getgatherProvider(): IGatherProvider | undefined { + public getGatherProvider(): IGatherProvider | undefined { const gatherLogger: INotebookExecutionLogger | undefined = this._loggers.find((logger: INotebookExecutionLogger) => { return (logger).service !== undefined; }); diff --git a/src/client/datascience/jupyter/liveshare/guestJupyterNotebook.ts b/src/client/datascience/jupyter/liveshare/guestJupyterNotebook.ts index e4bf77f5bd41..33fb2a83779f 100644 --- a/src/client/datascience/jupyter/liveshare/guestJupyterNotebook.ts +++ b/src/client/datascience/jupyter/liveshare/guestJupyterNotebook.ts @@ -206,7 +206,7 @@ export class GuestJupyterNotebook extends LiveShareParticipantGuest(LiveSharePar return; } - public getgatherProvider(): IGatherProvider | undefined { + public getGatherProvider(): IGatherProvider | undefined { return; } diff --git a/src/client/datascience/types.ts b/src/client/datascience/types.ts index 5ad3e20229b7..31f975d76878 100644 --- a/src/client/datascience/types.ts +++ b/src/client/datascience/types.ts @@ -110,7 +110,7 @@ export interface INotebook extends IAsyncDisposable { getKernelSpec(): IJupyterKernelSpec | LiveKernelModel | undefined; setKernelSpec(spec: IJupyterKernelSpec | LiveKernelModel, timeoutMS: number): Promise; setInterpreter(interpeter: PythonInterpreter): void; - getgatherProvider(): IGatherProvider | undefined; + getGatherProvider(): IGatherProvider | undefined; getCellHashProvider(): ICellHashProvider | undefined; } diff --git a/src/test/datascience/mockJupyterNotebook.ts b/src/test/datascience/mockJupyterNotebook.ts index a3e355876a92..20b301e50ce8 100644 --- a/src/test/datascience/mockJupyterNotebook.ts +++ b/src/test/datascience/mockJupyterNotebook.ts @@ -45,7 +45,7 @@ export class MockJupyterNotebook implements INotebook { constructor(private owner: INotebookServer) { noop(); } - public getgatherProvider(): IGatherProvider | undefined { + public getGatherProvider(): IGatherProvider | undefined { throw new Error('Method not implemented.'); } public getCellHashProvider(): ICellHashProvider | undefined { From 7d27f8df1092d924d2fb0b8464f910aba7916b19 Mon Sep 17 00:00:00 2001 From: Jim Griesmer Date: Fri, 24 Jan 2020 17:55:31 -0800 Subject: [PATCH 15/51] Private Gather support #1 Dynamically look for python-program-analysis. If not there gather is unsupported. --- package-lock.json | 5 - package.json | 1 - src/client/datascience/gather/gather.ts | 18 +- .../datascience/gather/gatherListener.ts | 12 +- src/client/datascience/serviceRegistry.ts | 15 +- .../datascience/dataScienceIocContainer.ts | 6 +- .../datascience/gather/gather.unit.test.ts | 36 +- .../cell.d.ts | 30 + .../cellslice.d.ts | 27 + .../control-flow.d.ts | 45 + .../data-flow.d.ts | 77 + .../genspec/main.d.ts | 1 + .../genspec/moduleSpecWalker.d.ts | 15 + .../graph.d.ts | 10 + .../index.d.ts | 12 + .../log-slicer.d.ts | 81 + .../logutil.d.ts | 2 + .../printNode.d.ts | 2 + .../program-builder.d.ts | 69 + .../python-parser.d.ts | 326 + .../rewrite-magics.d.ts | 82 + .../@msrvida-python-program-analysis/set.d.ts | 29 + .../slice.d.ts | 21 + .../specs/__builtins__.json | 69 + .../specs/index.d.ts | 35 + .../specs/matplotlib.json | 223 + .../specs/numpy.json | 540 + .../specs/pandas.json | 276 + .../specs/random.json | 28 + .../specs/sklearn.json | 13811 ++++++++++++++++ .../symbol-table.d.ts | 26 + .../test/analysis.test.d.ts | 1 + .../test/cellslice.test.d.ts | 1 + .../test/cfg.test.d.ts | 1 + .../test/graph.test.d.ts | 1 + .../test/log-slicer.test.d.ts | 1 + .../test/magics-rewriter.test.d.ts | 1 + .../test/merge.test.d.ts | 1 + ...03-feature-engineering-tfidf-expected.json | 155 + .../03-feature-engineering-tfidf.json | 404 + .../test/notebooks/Pima_Prediction.json | 1094 ++ .../test/notebooks/Titanic.json | 1315 ++ .../test/notebooks/Titanic2.json | 464 + .../notebooks/evaluating-models-expected.json | 165 + .../test/notebooks/evaluating-models.json | 174 + .../test/notebooks/index.d.ts | 17 + .../test/notebooks/vv.json | 306 + .../test/parser.test.d.ts | 1 + .../test/program-builder.test.d.ts | 1 + .../test/slice.test.d.ts | 1 + .../test/testcell.d.ts | 10 + 51 files changed, 20000 insertions(+), 44 deletions(-) create mode 100644 types/@msrvida-python-program-analysis/cell.d.ts create mode 100644 types/@msrvida-python-program-analysis/cellslice.d.ts create mode 100644 types/@msrvida-python-program-analysis/control-flow.d.ts create mode 100644 types/@msrvida-python-program-analysis/data-flow.d.ts create mode 100644 types/@msrvida-python-program-analysis/genspec/main.d.ts create mode 100644 types/@msrvida-python-program-analysis/genspec/moduleSpecWalker.d.ts create mode 100644 types/@msrvida-python-program-analysis/graph.d.ts create mode 100644 types/@msrvida-python-program-analysis/index.d.ts create mode 100644 types/@msrvida-python-program-analysis/log-slicer.d.ts create mode 100644 types/@msrvida-python-program-analysis/logutil.d.ts create mode 100644 types/@msrvida-python-program-analysis/printNode.d.ts create mode 100644 types/@msrvida-python-program-analysis/program-builder.d.ts create mode 100644 types/@msrvida-python-program-analysis/python-parser.d.ts create mode 100644 types/@msrvida-python-program-analysis/rewrite-magics.d.ts create mode 100644 types/@msrvida-python-program-analysis/set.d.ts create mode 100644 types/@msrvida-python-program-analysis/slice.d.ts create mode 100644 types/@msrvida-python-program-analysis/specs/__builtins__.json create mode 100644 types/@msrvida-python-program-analysis/specs/index.d.ts create mode 100644 types/@msrvida-python-program-analysis/specs/matplotlib.json create mode 100644 types/@msrvida-python-program-analysis/specs/numpy.json create mode 100644 types/@msrvida-python-program-analysis/specs/pandas.json create mode 100644 types/@msrvida-python-program-analysis/specs/random.json create mode 100644 types/@msrvida-python-program-analysis/specs/sklearn.json create mode 100644 types/@msrvida-python-program-analysis/symbol-table.d.ts create mode 100644 types/@msrvida-python-program-analysis/test/analysis.test.d.ts create mode 100644 types/@msrvida-python-program-analysis/test/cellslice.test.d.ts create mode 100644 types/@msrvida-python-program-analysis/test/cfg.test.d.ts create mode 100644 types/@msrvida-python-program-analysis/test/graph.test.d.ts create mode 100644 types/@msrvida-python-program-analysis/test/log-slicer.test.d.ts create mode 100644 types/@msrvida-python-program-analysis/test/magics-rewriter.test.d.ts create mode 100644 types/@msrvida-python-program-analysis/test/merge.test.d.ts create mode 100644 types/@msrvida-python-program-analysis/test/notebooks/03-feature-engineering-tfidf-expected.json create mode 100644 types/@msrvida-python-program-analysis/test/notebooks/03-feature-engineering-tfidf.json create mode 100644 types/@msrvida-python-program-analysis/test/notebooks/Pima_Prediction.json create mode 100644 types/@msrvida-python-program-analysis/test/notebooks/Titanic.json create mode 100644 types/@msrvida-python-program-analysis/test/notebooks/Titanic2.json create mode 100644 types/@msrvida-python-program-analysis/test/notebooks/evaluating-models-expected.json create mode 100644 types/@msrvida-python-program-analysis/test/notebooks/evaluating-models.json create mode 100644 types/@msrvida-python-program-analysis/test/notebooks/index.d.ts create mode 100644 types/@msrvida-python-program-analysis/test/notebooks/vv.json create mode 100644 types/@msrvida-python-program-analysis/test/parser.test.d.ts create mode 100644 types/@msrvida-python-program-analysis/test/program-builder.test.d.ts create mode 100644 types/@msrvida-python-program-analysis/test/slice.test.d.ts create mode 100644 types/@msrvida-python-program-analysis/test/testcell.d.ts diff --git a/package-lock.json b/package-lock.json index 4c5ee332b407..fc69cb95caf4 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1453,11 +1453,6 @@ "tinyqueue": "^1.1.0" } }, - "@msrvida/python-program-analysis": { - "version": "0.4.20", - "resolved": "https://pkgs.dev.azure.com/msresearch/_packaging/MSR-Python-Analysis/npm/registry/@msrvida/python-program-analysis/-/python-program-analysis-0.4.20.tgz", - "integrity": "sha1-NF+FuDLhWe7U5KkL48LDx8P1He8=" - }, "@nteract/markdown": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/@nteract/markdown/-/markdown-3.0.1.tgz", diff --git a/package.json b/package.json index db80560961fd..4e6dc0060da8 100644 --- a/package.json +++ b/package.json @@ -2801,7 +2801,6 @@ "@jupyterlab/services": "^4.2.0", "@koa/cors": "^3.0.0", "@loadable/component": "^5.12.0", - "@msrvida/python-program-analysis": "^0.4.1", "ansi-regex": "^4.1.0", "arch": "^2.1.0", "azure-storage": "^2.10.3", diff --git a/src/client/datascience/gather/gather.ts b/src/client/datascience/gather/gather.ts index 2f23930a7ff6..494022e07353 100644 --- a/src/client/datascience/gather/gather.ts +++ b/src/client/datascience/gather/gather.ts @@ -1,7 +1,6 @@ -import { Cell as IGatherCell, CellSlice, DataflowAnalyzer, ExecutionLogSlicer } from '@msrvida/python-program-analysis'; - import { inject, injectable } from 'inversify'; import * as uuid from 'uuid/v4'; +import { Cell, CellSlice, DataflowAnalyzer, ExecutionLogSlicer } from '../../../../types/@msrvida-python-program-analysis'; import { IApplicationShell, ICommandManager } from '../../common/application/types'; import { traceInfo } from '../../common/logger'; import { IConfigurationService, IDisposableRegistry } from '../../common/types'; @@ -11,12 +10,15 @@ import { Common } from '../../common/utils/localize'; import { Identifiers } from '../constants'; import { CellState, ICell as IVscCell, IGatherProvider } from '../types'; +// tslint:disable-next-line: no-var-requires no-require-imports +const programAnalysis: typeof import('@msrvida-python-program-analysis') = require('@msrvida-python-program-analysis'); + /** * An adapter class to wrap the code gathering functionality from [microsoft/python-program-analysis](https://www.npmjs.com/package/@msrvida/python-program-analysis). */ @injectable() -export class GatherExecution implements IGatherProvider { - private _executionSlicer: ExecutionLogSlicer; +export class GatherProvider implements IGatherProvider { + private _executionSlicer: ExecutionLogSlicer; private dataflowAnalyzer: DataflowAnalyzer; private _enabled: boolean; @@ -28,8 +30,8 @@ export class GatherExecution implements IGatherProvider { ) { this._enabled = this.configService.getSettings().datascience.enableGather ? true : false; - this.dataflowAnalyzer = new DataflowAnalyzer(); - this._executionSlicer = new ExecutionLogSlicer(this.dataflowAnalyzer); + this.dataflowAnalyzer = new programAnalysis.DataflowAnalyzer(); + this._executionSlicer = new programAnalysis.ExecutionLogSlicer(this.dataflowAnalyzer); if (this._enabled) { this.disposables.push(this.configService.getSettings().onDidChange(e => this.updateEnableGather(e))); @@ -108,10 +110,10 @@ function concat(existingText: string, newText: CellSlice): string { * This is called to convert VS Code ICells to Gather ICells for logging. * @param cell A cell object conforming to the VS Code cell interface */ -function convertVscToGatherCell(cell: IVscCell): IGatherCell | undefined { +function convertVscToGatherCell(cell: IVscCell): Cell | undefined { // This should always be true since we only want to log code cells. Putting this here so types match for outputs property if (cell.data.cell_type === 'code') { - const result: IGatherCell = { + const result: Cell = { // tslint:disable-next-line no-unnecessary-local-variable text: cell.data.source, diff --git a/src/client/datascience/gather/gatherListener.ts b/src/client/datascience/gather/gatherListener.ts index b671d6cf36af..3ce0ddeaa704 100644 --- a/src/client/datascience/gather/gatherListener.ts +++ b/src/client/datascience/gather/gatherListener.ts @@ -8,6 +8,7 @@ import { IFileSystem } from '../../common/platform/types'; import { IConfigurationService } from '../../common/types'; import * as localize from '../../common/utils/localize'; import { noop } from '../../common/utils/misc'; +import { IServiceContainer } from '../../ioc/types'; import { generateCellsFromString } from '../cellFactory'; import { Identifiers } from '../constants'; import { IInteractiveWindowMapping, InteractiveWindowMessages } from '../interactive-common/interactiveWindowTypes'; @@ -28,7 +29,8 @@ export class GatherListener implements IInteractiveWindowListener { @inject(IJupyterExecution) private jupyterExecution: IJupyterExecution, @inject(IInteractiveWindowProvider) private interactiveWindowProvider: IInteractiveWindowProvider, @inject(IDocumentManager) private documentManager: IDocumentManager, - @inject(IFileSystem) private fileSystem: IFileSystem + @inject(IFileSystem) private fileSystem: IFileSystem, + @inject(IServiceContainer) private serviceContainer: IServiceContainer ) { } public dispose() { @@ -69,9 +71,11 @@ export class GatherListener implements IInteractiveWindowListener { } private doGather(payload: ICell): void { - this.gatherCodeInternal(payload).catch(err => { - this.applicationShell.showErrorMessage(err); - }); + if (this.serviceContainer.getAll(IGatherProvider)) { + this.gatherCodeInternal(payload).catch(err => { + this.applicationShell.showErrorMessage(err); + }); + } } private doInitGather(payload: string): void { diff --git a/src/client/datascience/serviceRegistry.ts b/src/client/datascience/serviceRegistry.ts index ae8e9ba1e958..fa6d45248de4 100644 --- a/src/client/datascience/serviceRegistry.ts +++ b/src/client/datascience/serviceRegistry.ts @@ -21,7 +21,6 @@ import { DataScienceCodeLensProvider } from './editor-integration/codelensprovid import { CodeWatcher } from './editor-integration/codewatcher'; import { Decorator } from './editor-integration/decorator'; import { DataScienceErrorHandler } from './errorHandler/errorHandler'; -import { GatherExecution } from './gather/gather'; import { GatherListener } from './gather/gatherListener'; import { GatherLogger } from './gather/gatherLogger'; import { DebugListener } from './interactive-common/debugListener'; @@ -143,10 +142,6 @@ export function registerTypes(serviceManager: IServiceManager) { serviceManager.addBinding(ICellHashLogger, INotebookExecutionLogger); serviceManager.addBinding(ICellHashProvider, IInteractiveWindowListener); - serviceManager.add(IGatherProvider, GatherExecution); - serviceManager.add(IGatherLogger, GatherLogger); - serviceManager.addBinding(IGatherLogger, INotebookExecutionLogger); - serviceManager.addBinding(IJupyterDebugger, ICellHashListener); serviceManager.addSingleton(INotebookEditorProvider, NativeEditorProvider); serviceManager.add(INotebookEditor, NativeEditor); @@ -183,4 +178,14 @@ export function registerTypes(serviceManager: IServiceManager) { serviceManager.addSingleton(IJupyterSubCommandExecutionService, JupyterInterpreterSubCommandExecutionService); serviceManager.addSingleton(IJupyterInterpreterDependencyManager, JupyterInterpreterSubCommandExecutionService); } + + const gatherEnabled: boolean = false; //serviceManager.get('python.dataScience', undefined).getSettings().datascience.enableGather ? true : false; + if (gatherEnabled) { + // tslint:disable-next-line: no-require-imports + const gather = require('./gather/gather'); + + serviceManager.add(IGatherProvider, gather.GatherProvider); + serviceManager.add(IGatherLogger, GatherLogger); + serviceManager.addBinding(IGatherLogger, INotebookExecutionLogger); + } } diff --git a/src/test/datascience/dataScienceIocContainer.ts b/src/test/datascience/dataScienceIocContainer.ts index ba5a94678796..fd27ecd6ef2f 100644 --- a/src/test/datascience/dataScienceIocContainer.ts +++ b/src/test/datascience/dataScienceIocContainer.ts @@ -143,7 +143,7 @@ import { CodeLensFactory } from '../../client/datascience/editor-integration/cod import { DataScienceCodeLensProvider } from '../../client/datascience/editor-integration/codelensprovider'; import { CodeWatcher } from '../../client/datascience/editor-integration/codewatcher'; import { DataScienceErrorHandler } from '../../client/datascience/errorHandler/errorHandler'; -import { GatherExecution } from '../../client/datascience/gather/gather'; +import { GatherProvider } from '../../client/datascience/gather/gather'; import { GatherListener } from '../../client/datascience/gather/gatherListener'; import { IntellisenseProvider } from '../../client/datascience/interactive-common/intellisense/intellisenseProvider'; import { AutoSaveService } from '../../client/datascience/interactive-ipynb/autoSaveService'; @@ -477,7 +477,7 @@ export class DataScienceIocContainer extends UnitTestIocContainer { this.serviceManager.add(IInteractiveWindowListener, GatherListener); this.serviceManager.addBinding(ICellHashProvider, INotebookExecutionLogger); this.serviceManager.addBinding(IJupyterDebugger, ICellHashListener); - this.serviceManager.add(IGatherProvider, GatherExecution); + this.serviceManager.add(IGatherProvider, GatherProvider); this.serviceManager.addSingleton(ICodeLensFactory, CodeLensFactory); this.serviceManager.addSingleton(IShellDetector, TerminalNameShellDetector); this.serviceManager.addSingleton(InterpeterHashProviderFactory, InterpeterHashProviderFactory); @@ -928,7 +928,7 @@ export class DataScienceIocContainer extends UnitTestIocContainer { this.postMessageToWebPanel(msg); }, // tslint:disable-next-line:no-any no-empty - setState: (_msg: any) => {}, + setState: (_msg: any) => { }, // tslint:disable-next-line:no-any no-empty getState: () => { return {}; diff --git a/src/test/datascience/gather/gather.unit.test.ts b/src/test/datascience/gather/gather.unit.test.ts index 8b3e0568faf7..5b85b7559a3c 100644 --- a/src/test/datascience/gather/gather.unit.test.ts +++ b/src/test/datascience/gather/gather.unit.test.ts @@ -6,7 +6,7 @@ import * as assert from 'assert'; import * as TypeMoq from 'typemoq'; import { IApplicationShell, ICommandManager } from '../../../client/common/application/types'; import { IConfigurationService, IDataScienceSettings, IDisposableRegistry, IPythonSettings } from '../../../client/common/types'; -import { GatherExecution } from '../../../client/datascience/gather/gather'; +import { GatherProvider } from '../../../client/datascience/gather/gather'; import { GatherLogger } from '../../../client/datascience/gather/gatherLogger'; import { ICell as IVscCell } from '../../../client/datascience/types'; @@ -134,23 +134,23 @@ suite('DataScience code gathering unit tests', () => { pythonSettings.setup(p => p.datascience).returns(() => dataScienceSettings.object); configurationService.setup(c => c.getSettings(TypeMoq.It.isAny())).returns(() => pythonSettings.object); appShell.setup(a => a.showInformationMessage(TypeMoq.It.isAny(), TypeMoq.It.isAny())).returns(() => Promise.resolve('')); - const gatherExecution = new GatherExecution(configurationService.object, appShell.object, disposableRegistry.object, commandManager.object); - const gatherLogger = new GatherLogger(gatherExecution, configurationService.object); + // const GatherProvider = new GatherProvider(configurationService.object, appShell.object, disposableRegistry.object, commandManager.object); + // const gatherLogger = new GatherLogger(GatherProvider, configurationService.object); - test('Logs a cell execution', async () => { - let count = 0; - for (const c of codeCells) { - await gatherLogger.postExecute(c, false); - count += 1; - assert.equal(gatherExecution.executionSlicer.executionLog.length, count); - } - }); + // test('Logs a cell execution', async () => { + // let count = 0; + // for (const c of codeCells) { + // await gatherLogger.postExecute(c, false); + // count += 1; + // assert.equal(GatherProvider.executionSlicer.executionLog.length, count); + // } + // }); - test('Gathers program slices for a cell', async () => { - const defaultCellMarker = '# %%'; - const cell: IVscCell = codeCells[codeCells.length - 1]; - const program = gatherExecution.gatherCode(cell); - const expectedProgram = `# This file contains only the code required to produce the results of the gathered cell.\n${defaultCellMarker}\nfrom bokeh.plotting import show, figure, output_notebook\n\n${defaultCellMarker}\nx = [1,2,3,4,5]\ny = [21,9,15,17,4]\n\n${defaultCellMarker}\np=figure(title='demo',x_axis_label='x',y_axis_label='y')\n\n${defaultCellMarker}\np.line(x,y,line_width=2)\n\n${defaultCellMarker}\nshow(p)\n`; - assert.equal(program.trim(), expectedProgram.trim()); - }); + // test('Gathers program slices for a cell', async () => { + // const defaultCellMarker = '# %%'; + // const cell: IVscCell = codeCells[codeCells.length - 1]; + // const program = GatherProvider.gatherCode(cell); + // const expectedProgram = `# This file contains only the code required to produce the results of the gathered cell.\n${defaultCellMarker}\nfrom bokeh.plotting import show, figure, output_notebook\n\n${defaultCellMarker}\nx = [1,2,3,4,5]\ny = [21,9,15,17,4]\n\n${defaultCellMarker}\np=figure(title='demo',x_axis_label='x',y_axis_label='y')\n\n${defaultCellMarker}\np.line(x,y,line_width=2)\n\n${defaultCellMarker}\nshow(p)\n`; + // assert.equal(program.trim(), expectedProgram.trim()); + // }); }); diff --git a/types/@msrvida-python-program-analysis/cell.d.ts b/types/@msrvida-python-program-analysis/cell.d.ts new file mode 100644 index 000000000000..4f0bde607a67 --- /dev/null +++ b/types/@msrvida-python-program-analysis/cell.d.ts @@ -0,0 +1,30 @@ +/** + * Generic interface for accessing data about a code cell. + */ +export interface Cell { + /** + * The cell's current text. + */ + text: string; + executionCount: number; + /** + * A unique ID generated each time a cell is executed. This lets us disambiguate between two + * runs of a cell that have the same ID *and* execution count, if the kernel was restarted. + * This ID should also be programmed to be *persistent*, so that even after a notebook is + * reloaded, the cell in the same position will still have this ID. + */ + readonly executionEventId: string; + /** + * A persistent ID for a cell in a notebook. This ID will stay the same even as the cell is + * executed, and even when the cell is reloaded from the file. + */ + readonly persistentId: string; + /** + * Whether analysis or execution of this cell has yielded an error. + */ + hasError: boolean; + /** + * Create a deep copy of the cell. + */ + deepCopy: () => Cell; +} diff --git a/types/@msrvida-python-program-analysis/cellslice.d.ts b/types/@msrvida-python-program-analysis/cellslice.d.ts new file mode 100644 index 000000000000..0c2d9060ebec --- /dev/null +++ b/types/@msrvida-python-program-analysis/cellslice.d.ts @@ -0,0 +1,27 @@ +import { Cell } from './cell'; +import { LocationSet } from './slice'; +export declare class CellSlice { + /** + * Construct an instance of a cell slice. + */ + constructor(cell: Cell, slice: LocationSet, executionTime?: Date); + /** + * Get the text in the slice of a cell. + */ + readonly textSlice: string; + /** + * Get the text of all lines in a slice (no deletions from lines). + */ + readonly textSliceLines: string; + private getTextSlice; + /** + * Get the slice. + */ + /** + * Set the slice. + */ + slice: LocationSet; + readonly cell: Cell; + readonly executionTime: Date; + private _slice; +} diff --git a/types/@msrvida-python-program-analysis/control-flow.d.ts b/types/@msrvida-python-program-analysis/control-flow.d.ts new file mode 100644 index 000000000000..f08166505bcb --- /dev/null +++ b/types/@msrvida-python-program-analysis/control-flow.d.ts @@ -0,0 +1,45 @@ +import * as ast from './python-parser'; +export declare class Block { + id: number; + readonly hint: string; + statements: ast.SyntaxNode[]; + loopVariables: ast.SyntaxNode[]; + constructor(id: number, hint: string, statements: ast.SyntaxNode[], loopVariables?: ast.SyntaxNode[]); + toString(): string; +} +export declare class ControlFlowGraph { + private _blocks; + private globalId; + private entry; + private exit; + private successors; + private loopVariables; + constructor(node: ast.SyntaxNode); + private makeBlock; + readonly blocks: Block[]; + getSuccessors(block: Block): Block[]; + getPredecessors(block: Block): Block[]; + print(): void; + private link; + private handleIf; + private handleWhile; + private handleFor; + private handleWith; + private handleTry; + private makeCFG; + /** + * Based on the algorithm in "Engineering a Compiler", 2nd ed., Cooper and Torczon: + * - p479: computing dominance + * - p498-500: dominator trees and frontiers + * - p544: postdominance and reverse dominance frontier + */ + visitControlDependencies(visit: (controlStmt: ast.SyntaxNode, stmt: ast.SyntaxNode) => void): void; + private postdominators; + private immediatePostdominators; + private reverseDominanceFrontiers; + private postdominatorExists; + private getImmediatePostdominator; + private findPostdominators; + private getImmediatePostdominators; + private buildReverseDominanceFrontiers; +} diff --git a/types/@msrvida-python-program-analysis/data-flow.d.ts b/types/@msrvida-python-program-analysis/data-flow.d.ts new file mode 100644 index 000000000000..025426f113c9 --- /dev/null +++ b/types/@msrvida-python-program-analysis/data-flow.d.ts @@ -0,0 +1,77 @@ +import * as ast from './python-parser'; +import { ControlFlowGraph } from './control-flow'; +import { Set } from './set'; +import { JsonSpecs, PythonType } from './specs'; +declare class DefUse { + DEFINITION: RefSet; + UPDATE: RefSet; + USE: RefSet; + constructor(DEFINITION?: RefSet, UPDATE?: RefSet, USE?: RefSet); + readonly defs: Set; + readonly uses: Set; + union(that: DefUse): DefUse; + update(newRefs: DefUse): void; + equals(that: DefUse): boolean; + createFlowsFrom(fromSet: DefUse): [Set, Set]; +} +/** + * Use a shared dataflow analyzer object for all dataflow analysis / querying for defs and uses. + * It caches defs and uses for each statement, which can save time. + * For caching to work, statements must be annotated with a cell's ID and execution count. + */ +export declare class DataflowAnalyzer { + constructor(moduleMap?: JsonSpecs); + getDefUseForStatement(statement: ast.SyntaxNode, defsForMethodResolution: RefSet): DefUse; + analyze(cfg: ControlFlowGraph, refSet?: RefSet): DataflowAnalysisResult; + getDefs(statement: ast.SyntaxNode, defsForMethodResolution: RefSet): RefSet; + private getClassDefs; + private getFuncDefs; + private getAssignDefs; + private getDelDefs; + private getFromImportDefs; + private getImportDefs; + getUses(statement: ast.SyntaxNode): RefSet; + private getNameUses; + private getClassDeclUses; + private getFuncDeclUses; + private getAssignUses; + private _symbolTable; + private _defUsesCache; +} +export interface Dataflow { + fromNode: ast.SyntaxNode; + toNode: ast.SyntaxNode; + fromRef?: Ref; + toRef?: Ref; +} +export declare enum ReferenceType { + DEFINITION = "DEFINITION", + UPDATE = "UPDATE", + USE = "USE" +} +export declare enum SymbolType { + VARIABLE = 0, + CLASS = 1, + FUNCTION = 2, + IMPORT = 3, + MUTATION = 4, + MAGIC = 5 +} +export interface Ref { + type: SymbolType; + level: ReferenceType; + name: string; + inferredType?: PythonType; + location: ast.Location; + node: ast.SyntaxNode; +} +export declare class RefSet extends Set { + constructor(...items: Ref[]); +} +export declare function sameLocation(loc1: ast.Location, loc2: ast.Location): boolean; +export declare const GlobalSyntheticVariable = "$global"; +export declare type DataflowAnalysisResult = { + dataflows: Set; + undefinedRefs: RefSet; +}; +export {}; diff --git a/types/@msrvida-python-program-analysis/genspec/main.d.ts b/types/@msrvida-python-program-analysis/genspec/main.d.ts new file mode 100644 index 000000000000..cb0ff5c3b541 --- /dev/null +++ b/types/@msrvida-python-program-analysis/genspec/main.d.ts @@ -0,0 +1 @@ +export {}; diff --git a/types/@msrvida-python-program-analysis/genspec/moduleSpecWalker.d.ts b/types/@msrvida-python-program-analysis/genspec/moduleSpecWalker.d.ts new file mode 100644 index 000000000000..31c9a03d7e90 --- /dev/null +++ b/types/@msrvida-python-program-analysis/genspec/moduleSpecWalker.d.ts @@ -0,0 +1,15 @@ +import * as py from '../python-parser'; +import { ModuleSpec, FunctionDescription } from ".."; +export declare class ModuleSpecWalker { + spec: ModuleSpec; + constructor(); + private static lookForSideEffects; + onEnterNode(node: py.SyntaxNode, ancestors: py.SyntaxNode[]): void; +} +export declare class HeuristicTransitiveClosure { + private moduleSpec; + constructor(moduleSpec: ModuleSpec); + private transferSideEffectsAcrossCalls; + private recordSideEffects; + onEnterNode(node: py.SyntaxNode, ancestors: py.SyntaxNode[]): void; +} diff --git a/types/@msrvida-python-program-analysis/graph.d.ts b/types/@msrvida-python-program-analysis/graph.d.ts new file mode 100644 index 000000000000..547c3ae01fa8 --- /dev/null +++ b/types/@msrvida-python-program-analysis/graph.d.ts @@ -0,0 +1,10 @@ +export declare class Graph { + private getIdentifier; + private outgoing; + private incoming; + private _nodes; + constructor(getIdentifier: (item: T) => string); + addEdge(fromNode: T, toNode: T): void; + readonly nodes: T[]; + topoSort(): T[]; +} diff --git a/types/@msrvida-python-program-analysis/index.d.ts b/types/@msrvida-python-program-analysis/index.d.ts new file mode 100644 index 000000000000..5e7fcce01946 --- /dev/null +++ b/types/@msrvida-python-program-analysis/index.d.ts @@ -0,0 +1,12 @@ +export * from './set'; +export * from './python-parser'; +export * from './control-flow'; +export * from './data-flow'; +export * from './printNode'; +export * from './specs'; +export * from './cell'; +export * from './slice'; +export * from './cellslice'; +export * from './log-slicer'; +export * from './program-builder'; +export * from './specs/index'; diff --git a/types/@msrvida-python-program-analysis/log-slicer.d.ts b/types/@msrvida-python-program-analysis/log-slicer.d.ts new file mode 100644 index 000000000000..dbe1173aca8d --- /dev/null +++ b/types/@msrvida-python-program-analysis/log-slicer.d.ts @@ -0,0 +1,81 @@ +import { Cell } from './cell'; +import { CellSlice } from './cellslice'; +import { DataflowAnalyzer } from './data-flow'; +import { CellProgram, ProgramBuilder } from './program-builder'; +import { LocationSet } from './slice'; +/** + * A record of when a cell was executed. + */ +export declare class CellExecution { + readonly cell: TCell; + readonly executionTime: Date; + constructor(cell: TCell, executionTime: Date); + /** + * Update this method if at some point we only want to save some about a CellExecution when + * serializing it and saving history. + */ + toJSON(): any; +} +/** + * A slice over a version of executed code. + */ +export declare class SlicedExecution { + executionTime: Date; + cellSlices: CellSlice[]; + constructor(executionTime: Date, cellSlices: CellSlice[]); + merge(...slicedExecutions: SlicedExecution[]): SlicedExecution; +} +export declare type CellExecutionCallback = (exec: CellExecution) => void; +/** + * Makes slice on a log of executed cells. + */ +export declare class ExecutionLogSlicer { + private dataflowAnalyzer; + executionLog: CellExecution[]; + readonly programBuilder: ProgramBuilder; + /** + * Signal emitted when a cell's execution has been completely processed. + */ + readonly executionLogged: CellExecutionCallback[]; + /** + * Construct a new execution log slicer. + */ + constructor(dataflowAnalyzer: DataflowAnalyzer); + /** + * Log that a cell has just been executed. The execution time for this cell will be stored + * as the moment at which this method is called. + */ + logExecution(cell: TCell): void; + /** + * Use logExecution instead if a cell has just been run to annotate it with the current time + * as the execution time. This function is intended to be used only to initialize history + * when a notebook is reloaded. However, any method that eventually calls this method will + * notify all observers that this cell has been executed. + */ + addExecutionToLog(cellExecution: CellExecution): void; + /** + * Reset the log, removing log records. + */ + reset(): void; + /** + * Get slice for the latest execution of a cell. + */ + sliceLatestExecution(cellId: string, seedLocations?: LocationSet): SlicedExecution; + /** + * Get slices of the necessary code for all executions of a cell. + * Relevant line numbers are relative to the cell's start line (starting at first line = 0). + */ + sliceAllExecutions(cellId: string, seedLocations?: LocationSet): SlicedExecution[]; + readonly cellExecutions: ReadonlyArray>; + /** + * Get the cell program (tree, defs, uses) for a cell. + */ + getCellProgram(executionEventId: string): CellProgram; + /** + * Returns the cells that directly or indirectly use variables + * that are defined in the given cell. Result is in + * topological order. + * @param executionEventId a cell in the log + */ + getDependentCells(executionEventId: string): Cell[]; +} diff --git a/types/@msrvida-python-program-analysis/logutil.d.ts b/types/@msrvida-python-program-analysis/logutil.d.ts new file mode 100644 index 000000000000..e0d5a773d351 --- /dev/null +++ b/types/@msrvida-python-program-analysis/logutil.d.ts @@ -0,0 +1,2 @@ +export declare function startLogging(): void; +export declare function log(message: string, ...args: any[]): void; diff --git a/types/@msrvida-python-program-analysis/printNode.d.ts b/types/@msrvida-python-program-analysis/printNode.d.ts new file mode 100644 index 000000000000..3f6b1a32b4c5 --- /dev/null +++ b/types/@msrvida-python-program-analysis/printNode.d.ts @@ -0,0 +1,2 @@ +import { SyntaxNode } from './python-parser'; +export declare function printNode(node: SyntaxNode): string; diff --git a/types/@msrvida-python-program-analysis/program-builder.d.ts b/types/@msrvida-python-program-analysis/program-builder.d.ts new file mode 100644 index 000000000000..af4eac682017 --- /dev/null +++ b/types/@msrvida-python-program-analysis/program-builder.d.ts @@ -0,0 +1,69 @@ +import { Cell } from './cell'; +import * as ast from './python-parser'; +import { DataflowAnalyzer, Ref } from './data-flow'; +import { NumberSet } from './set'; +/** + * Maps to find out what line numbers over a program correspond to what cells. + */ +export declare type CellToLineMap = { + [cellExecutionEventId: string]: NumberSet; +}; +export declare type LineToCellMap = { + [line: number]: Cell; +}; +/** + * A program built from cells. + */ +export declare class Program { + /** + * Construct a program. + */ + constructor(cellPrograms: CellProgram[]); + readonly text: string; + readonly tree: ast.Module; + readonly cellToLineMap: CellToLineMap; + readonly lineToCellMap: LineToCellMap; +} +/** + * Program fragment for a cell. Used to cache parsing results. + */ +export declare class CellProgram { + /** + * Construct a cell program + */ + constructor(cell: Cell, statements: ast.SyntaxNode[], defs: Ref[], uses: Ref[], hasError: boolean); + readonly cell: Cell; + readonly statements: ast.SyntaxNode[]; + readonly defs: Ref[]; + readonly uses: Ref[]; + readonly hasError: boolean; + usesSomethingFrom(that: CellProgram): boolean; +} +/** + * Builds programs from a list of executed cells. + */ +export declare class ProgramBuilder { + /** + * Construct a program builder. + */ + constructor(dataflowAnalyzer?: DataflowAnalyzer); + /** + * Add cells to the program builder. + */ + add(...cells: Cell[]): void; + /** + * Reset (removing all cells). + */ + reset(): void; + /** + * Build a program from the list of cells. Program will include the cells' contents in + * the order they were added to the log. It will omit cells that raised errors (syntax or + * runtime, except for the last cell). + */ + buildTo(cellExecutionEventId: string): Program; + buildFrom(executionEventId: string): Program; + getCellProgram(executionEventId: string): CellProgram; + getCellProgramsWithSameId(executionEventId: string): CellProgram[]; + private _cellPrograms; + private _dataflowAnalyzer; +} diff --git a/types/@msrvida-python-program-analysis/python-parser.d.ts b/types/@msrvida-python-program-analysis/python-parser.d.ts new file mode 100644 index 000000000000..d205d4f3a597 --- /dev/null +++ b/types/@msrvida-python-program-analysis/python-parser.d.ts @@ -0,0 +1,326 @@ +/** + * This is the main interface for parsing code. + * Call this instead of the `parse` method in python3.js. + * If the `parse` method gets an error, all later calls will throw an error. + * This method resets the state of the `parse` method so that doesn't happen. + */ +export declare function parse(program: string): Module; +export declare type SyntaxNode = Module | Import | From | Decorator | Decorate | Def | Parameter | Assignment | Delete | Assert | Pass | Return | Yield | Raise | Continue | Break | Global | Nonlocal | If | Else | While | For | Try | With | Call | Index | Slice | Dot | IfExpr | CompFor | CompIf | Lambda | UnaryOperator | BinaryOperator | Starred | Tuple | ListExpr | SetExpr | DictExpr | Name | Literal | Class | Argument; +interface JisonLocation { + first_line: number; + first_column: number; + last_line: number; + last_column: number; +} +export interface Location extends JisonLocation { + path?: string; +} +export declare function locationString(loc: Location): string; +export declare function locationContains(loc1: Location, loc2: Location): boolean; +export interface Locatable { + location: Location; + cellId?: string; + executionCount?: number; +} +export declare const MODULE = "module"; +export interface Module extends Locatable { + type: typeof MODULE; + code: SyntaxNode[]; +} +export declare const IMPORT = "import"; +export interface Import extends Locatable { + type: typeof IMPORT; + names: { + path: string; + alias?: string; + location: Location; + }[]; +} +export declare const FROM = "from"; +export interface From extends Locatable { + type: typeof FROM; + base: string; + imports: { + path: string; + alias: string; + location: Location; + }[]; +} +export declare const DECORATOR = "decorator"; +export interface Decorator extends Locatable { + type: typeof DECORATOR; + decorator: string; + args: SyntaxNode[]; +} +export declare const DECORATE = "decorate"; +export interface Decorate extends Locatable { + type: typeof DECORATE; + decorators: Decorator[]; + def: SyntaxNode; +} +export declare const DEF = "def"; +export interface Def extends Locatable { + type: typeof DEF; + name: string; + params: Parameter[]; + code: SyntaxNode[]; +} +export declare const PARAMETER = "parameter"; +export interface Parameter extends Locatable { + type: typeof PARAMETER; + name: string; + anno: SyntaxNode; + default_value: SyntaxNode; + star: boolean; + starstar: boolean; +} +export declare const ASSIGN = "assign"; +export interface Assignment extends Locatable { + type: typeof ASSIGN; + op: string | undefined; + targets: SyntaxNode[]; + sources: SyntaxNode[]; +} +export declare const DEL = "del"; +export interface Delete extends Locatable { + type: typeof DEL; + targets: SyntaxNode[]; +} +export declare const ASSERT = "assert"; +export interface Assert extends Locatable { + type: typeof ASSERT; + cond: SyntaxNode; + err: SyntaxNode; +} +export declare const PASS = "pass"; +export interface Pass extends Locatable { + type: typeof PASS; +} +export declare const RETURN = "return"; +export interface Return extends Locatable { + type: typeof RETURN; + values: SyntaxNode[]; +} +export declare const YIELD = "yield"; +export interface Yield extends Locatable { + type: typeof YIELD; + value: SyntaxNode[]; + from?: SyntaxNode; +} +export declare const RAISE = "raise"; +export interface Raise extends Locatable { + type: typeof RAISE; + err: SyntaxNode; +} +export declare const BREAK = "break"; +export interface Break extends Locatable { + type: typeof BREAK; +} +export declare const CONTINUE = "continue"; +export interface Continue extends Locatable { + type: typeof CONTINUE; +} +export declare const GLOBAL = "global"; +export interface Global extends Locatable { + type: typeof GLOBAL; + names: string[]; +} +export declare const NONLOCAL = "nonlocal"; +export interface Nonlocal extends Locatable { + type: typeof NONLOCAL; + names: string[]; +} +export declare const IF = "if"; +export interface If extends Locatable { + type: typeof IF; + cond: SyntaxNode; + code: SyntaxNode[]; + elif: { + cond: SyntaxNode; + code: SyntaxNode[]; + }[]; + else: Else; +} +export declare const WHILE = "while"; +export interface While extends Locatable { + type: typeof WHILE; + cond: SyntaxNode; + code: SyntaxNode[]; + else: SyntaxNode[]; +} +export declare const ELSE = "else"; +export interface Else extends Locatable { + type: typeof ELSE; + code: SyntaxNode[]; +} +export declare const FOR = "for"; +export interface For extends Locatable { + type: typeof FOR; + target: SyntaxNode[]; + iter: SyntaxNode[]; + code: SyntaxNode[]; + else?: SyntaxNode[]; + decl_location: Location; +} +export declare const COMPFOR = "comp_for"; +export interface CompFor extends Locatable { + type: typeof COMPFOR; + for: SyntaxNode[]; + in: SyntaxNode; +} +export declare const COMPIF = "comp_if"; +export interface CompIf extends Locatable { + type: typeof COMPIF; + test: SyntaxNode; +} +export declare const TRY = "try"; +export interface Try extends Locatable { + type: typeof TRY; + code: SyntaxNode[]; + excepts: { + cond: SyntaxNode; + name: string; + code: SyntaxNode[]; + }[]; + else: SyntaxNode[]; + finally: SyntaxNode[]; +} +export declare const WITH = "with"; +export interface With extends Locatable { + type: typeof WITH; + items: { + with: SyntaxNode; + as: SyntaxNode; + }[]; + code: SyntaxNode[]; +} +export declare const CALL = "call"; +export interface Call extends Locatable { + type: typeof CALL; + func: SyntaxNode; + args: Argument[]; +} +export declare const ARG = "arg"; +export interface Argument extends Locatable { + type: typeof ARG; + actual: SyntaxNode; + keyword?: SyntaxNode; + loop?: CompFor; + varargs?: boolean; + kwargs?: boolean; +} +export declare const INDEX = "index"; +export interface Index extends Locatable { + type: typeof INDEX; + value: SyntaxNode; + args: SyntaxNode[]; +} +export declare const SLICE = "slice"; +export interface Slice extends Locatable { + type: typeof SLICE; + start?: SyntaxNode; + stop?: SyntaxNode; + step?: SyntaxNode; +} +export declare const DOT = "dot"; +export interface Dot extends Locatable { + type: typeof DOT; + value: SyntaxNode; + name: string; +} +export declare const IFEXPR = "ifexpr"; +export interface IfExpr extends Locatable { + type: typeof IFEXPR; + test: SyntaxNode; + then: SyntaxNode; + else: SyntaxNode; +} +export declare const LAMBDA = "lambda"; +export interface Lambda extends Locatable { + type: typeof LAMBDA; + args: Parameter[]; + code: SyntaxNode; +} +export declare const UNOP = "unop"; +export interface UnaryOperator extends Locatable { + type: typeof UNOP; + op: string; + operand: SyntaxNode; +} +export declare const BINOP = "binop"; +export interface BinaryOperator extends Locatable { + type: typeof BINOP; + op: string; + left: SyntaxNode; + right: SyntaxNode; +} +export declare const STARRED = "starred"; +export interface Starred extends Locatable { + type: typeof STARRED; + value: SyntaxNode; +} +export declare const TUPLE = "tuple"; +export interface Tuple extends Locatable { + type: typeof TUPLE; + items: SyntaxNode[]; +} +export declare const LIST = "list"; +export interface ListExpr extends Locatable { + type: typeof LIST; + items: SyntaxNode[]; +} +export declare const SET = "set"; +export interface SetExpr extends Locatable { + type: typeof SET; + entries: SyntaxNode[]; + comp_for?: SyntaxNode[]; +} +export declare const DICT = "dict"; +export interface DictExpr extends Locatable { + type: typeof DICT; + entries: { + k: SyntaxNode; + v: SyntaxNode; + }[]; + comp_for?: SyntaxNode[]; +} +export declare const NAME = "name"; +export interface Name extends Locatable { + type: typeof NAME; + id: string; +} +export declare const LITERAL = "literal"; +export interface Literal extends Locatable { + type: typeof LITERAL; + value: any; +} +export declare const CLASS = "class"; +export interface Class extends Locatable { + type: typeof CLASS; + name: string; + extends: SyntaxNode[]; + code: SyntaxNode[]; +} +/** + * returns whether two syntax nodes are semantically equivalent + */ +export declare function isEquivalent(node1: SyntaxNode, node2: SyntaxNode): boolean; +export declare function flatten(arrayArrays: T[][]): T[]; +/** + * Listener for pre-order traversal of the parse tree. + */ +export interface WalkListener { + /** + * Called whenever a node is entered. + */ + onEnterNode?(node: SyntaxNode, ancestors: SyntaxNode[]): void; + /** + * Called whenever a node is exited. + */ + onExitNode?(node: SyntaxNode, ancestors: SyntaxNode[]): void; +} +/** + * Preorder tree traversal with optional listener. + */ +export declare function walk(node: SyntaxNode, walkListener?: WalkListener): SyntaxNode[]; +export {}; diff --git a/types/@msrvida-python-program-analysis/rewrite-magics.d.ts b/types/@msrvida-python-program-analysis/rewrite-magics.d.ts new file mode 100644 index 000000000000..5c19672413ae --- /dev/null +++ b/types/@msrvida-python-program-analysis/rewrite-magics.d.ts @@ -0,0 +1,82 @@ +/** + * Result of rewriting a magic line. + */ +export declare type Rewrite = { + text?: string; + annotations?: MagicAnnotation[]; +}; +/** + * An annotation to hold metadata about what a magic is doing. + */ +export declare type MagicAnnotation = { + key: string; + value: string; +}; +/** + * Position of a text match for magics. + */ +export declare type MatchPosition = [{ + line: number; + col: number; +}, { + line: number; + col: number; +}]; +/** + * Interface for command-specific magic rewrites. + */ +export interface LineMagicRewriter { + /** + * Name of the magic command this will apply to. + */ + commandName: string; + /** + * Rewrite the line magic. + * @param matchedText the original matched text from the program + * @param magicStmt the line magic text with newlines and continuations removed + * @param postion ((start_line, start_col),(end_line, end_col)) of `matchedText` within the cell + * @return rewrite operation. Leave text empty if you want to use default rewrites. + */ + rewrite(matchedText: string, magicStmt: string, position: MatchPosition): Rewrite; +} +/** + * Utility to rewrite IPython code to remove magics. + * Should be applied at to cells, not the entire program, to properly handle cell magics. + * One of the most important aspects of the rewriter is that it shouldn't change the line number + * of any of the statements in the program. If it does, this will make it impossible to + * map back from the results of code analysis to the relevant code in the editor. + */ +export declare class MagicsRewriter { + /** + * Construct a magics rewriter. + */ + constructor(lineMagicRewriters?: LineMagicRewriter[]); + /** + * Rewrite code so that it doesn't contain magics. + */ + rewrite(text: string, lineMagicRewriters?: LineMagicRewriter[]): string; + /** + * Default rewrite rule for cell magics. + */ + rewriteCellMagic(text: string): string; + /** + * Default rewrite rule for line magics. + */ + rewriteLineMagic(text: string, lineMagicRewriters?: LineMagicRewriter[]): string; + private _lineMagicRewriters; + private _defaultLineMagicRewriters; +} +/** + * Line magic rewriter for the "time" magic. + */ +export declare class TimeLineMagicRewriter implements LineMagicRewriter { + commandName: string; + rewrite(matchedText: string, magicStmt: string, position: MatchPosition): Rewrite; +} +/** + * Line magic rewriter for the "pylab" magic. + */ +export declare class PylabLineMagicRewriter implements LineMagicRewriter { + commandName: string; + rewrite(matchedText: string, magicStmt: string, position: MatchPosition): Rewrite; +} diff --git a/types/@msrvida-python-program-analysis/set.d.ts b/types/@msrvida-python-program-analysis/set.d.ts new file mode 100644 index 000000000000..ef87d343dc75 --- /dev/null +++ b/types/@msrvida-python-program-analysis/set.d.ts @@ -0,0 +1,29 @@ +export declare class Set { + private getIdentifier; + private _items; + constructor(getIdentifier: (item: T) => string, ...items: T[]); + readonly size: number; + add(...items: T[]): void; + remove(item: T): void; + pop(): T; + has(item: T): boolean; + readonly items: T[]; + equals(that: Set): boolean; + readonly empty: boolean; + union(...those: Set[]): Set; + intersect(that: Set): Set; + filter(predicate: (item: T) => boolean): Set; + map(getIdentifier: (item: U) => string, transform: (item: T) => U): Set; + mapSame(transform: (item: T) => T): Set; + some(predicate: (item: T) => boolean): boolean; + minus(that: Set): Set; + take(): T; + product(that: Set): Set<[T, T]>; +} +export declare class StringSet extends Set { + constructor(...items: string[]); +} +export declare class NumberSet extends Set { + constructor(...items: number[]); +} +export declare function range(min: number, max: number): Set; diff --git a/types/@msrvida-python-program-analysis/slice.d.ts b/types/@msrvida-python-program-analysis/slice.d.ts new file mode 100644 index 000000000000..d51a768f6ba1 --- /dev/null +++ b/types/@msrvida-python-program-analysis/slice.d.ts @@ -0,0 +1,21 @@ +import { Location, Module } from './python-parser'; +import { DataflowAnalyzer } from './data-flow'; +import { NumberSet, Set } from './set'; +export declare class LocationSet extends Set { + constructor(...items: Location[]); +} +export declare enum SliceDirection { + Forward = 0, + Backward = 1 +} +/** + * More general slice: given locations of important syntax nodes, find locations of all relevant + * definitions. Locations can be mapped to lines later. + * seedLocations are symbol locations. + */ +export declare function slice(ast: Module, seedLocations?: LocationSet, dataflowAnalyzer?: DataflowAnalyzer, direction?: SliceDirection): LocationSet; +/** + * Slice: given a set of lines in a program, return lines it depends on. + * OUT OF DATE: use slice() instead of sliceLines(). + */ +export declare function sliceLines(code: string, relevantLineNumbers: NumberSet): NumberSet; diff --git a/types/@msrvida-python-program-analysis/specs/__builtins__.json b/types/@msrvida-python-program-analysis/specs/__builtins__.json new file mode 100644 index 000000000000..1969d19e3bc5 --- /dev/null +++ b/types/@msrvida-python-program-analysis/specs/__builtins__.json @@ -0,0 +1,69 @@ +{ + "__builtins__": { + "functions": [ + "credits", + { "name": "delattr", "updates": [1] }, + "dict", + "dir", + "divmod", + "enumerate", + "eval", + "exec", + "exit", + { "name": "filter", "higherorder": 1 }, + "float", + "format", + "frozenset", + "getattr", + "globals", + "hasattr", + "hash", + "help", + "hex", + "id", + "input", + "int", + "isinstance", + "issubclass", + "iter", + "len", + "license", + "list", + "locals", + { "name": "map", "higherorder": 1 }, + "max", + "memoryview", + "min", + "next", + "object", + "oct", + "open", + "ord", + "pow", + "print", + "property", + "quit", + "range", + "repr", + "reversed", + "round", + "set", + { "name": "setattr", "updates": [1] }, + "slice", + "sorted", + "staticmethod", + "str", + "sum", + "super", + "tuple", + "type", + "vars", + "zip" + ], + "types": { + "BaseException": { + "methods": [{ "name": "with_traceback", "updates": [1] }] + } + } + } +} diff --git a/types/@msrvida-python-program-analysis/specs/index.d.ts b/types/@msrvida-python-program-analysis/specs/index.d.ts new file mode 100644 index 000000000000..c459828998c8 --- /dev/null +++ b/types/@msrvida-python-program-analysis/specs/index.d.ts @@ -0,0 +1,35 @@ +export interface FunctionSpec { + name: string; + updates?: (string | number)[]; + reads?: string[]; + returns?: string; + returnsType?: PythonType; + higherorder?: number; +} +export declare type FunctionDescription = string | FunctionSpec; +export declare function getFunctionName(fd: FunctionDescription): string; +export declare function isFunctionSpec(fd: FunctionDescription): fd is FunctionSpec; +export declare type PythonType = ListType | ClassType; +export declare class ListType { + elementType: PythonType; + constructor(elementType: PythonType); +} +export declare class ClassType { + spec: TypeSpec; + constructor(spec: TypeSpec); +} +export interface TypeSpec { + methods?: FD[]; +} +export interface ModuleSpec extends TypeSpec { + functions?: FD[]; + modules?: ModuleMap; + types?: { + [typeName: string]: TypeSpec; + }; +} +export interface ModuleMap { + [moduleName: string]: ModuleSpec; +} +export declare type JsonSpecs = ModuleMap; +export declare const DefaultSpecs: JsonSpecs; diff --git a/types/@msrvida-python-program-analysis/specs/matplotlib.json b/types/@msrvida-python-program-analysis/specs/matplotlib.json new file mode 100644 index 000000000000..cb8513693dfb --- /dev/null +++ b/types/@msrvida-python-program-analysis/specs/matplotlib.json @@ -0,0 +1,223 @@ +{ + "matplotlib": { + "functions": [ + "checkdep_dvipng", + "checkdep_ghostscript", + "checkdep_inkscape", + "checkdep_pdftops", + "checkdep_ps_distiller", + "checkdep_usetex", + "compare_versions", + "cycler", + "dedent", + { "name": "get_backend", "reads": ["rcParams"] }, + "get_cachedir", + "get_configdir", + "get_data_path", + "get_home", + "get_label", + "get_py2exe_datafiles", + { "name": "interactive", "updates": ["rcParams"] }, + { "name": "is_interactive", "reads": ["rcParams"] }, + "is_url", + "matplotlib_fname", + "mplDeprecation", + { "name": "rc", "updates": ["rcParams"] }, + { "name": "rc_context", "updates": ["rcParams"] }, + { "name": "rc_file", "updates": ["rcParams"] }, + { "name": "rc_file_defaults", "updates": ["rcParams"] }, + "rc_params", + "rc_params_from_file", + { "name": "rcdefaults", "updates": ["rcParams"] }, + "sanitize_sequence", + "test", + { "name": "tk_window_focus", "updates": ["rcParams"] }, + "use", + "validate_backend" + ], + "modules": { + "pyplot": { + "functions": [ + "acorr", + "angle_spectrum", + "annotate", + "arrow", + "autoscale", + "autumn", + "axes", + "axhline", + "axhspan", + "axis", + "axvline", + "axvspan", + "bar", + "barbs", + "barh", + "bone", + "box", + "boxplot", + "broken_barh", + "cla", + "clabel", + "clf", + "clim", + "close", + "cm", + "cohere", + "colorbar", + "colormaps", + "connect", + "contour", + "contourf", + "cool", + "copper", + "csd", + "cycler", + "dedent", + "delaxes", + "deprecated", + "disconnect", + "docstring", + "draw", + "draw_all", + "draw_if_interactive", + "errorbar", + "eventplot", + "figaspect", + "figimage", + "figlegend", + "fignum_exists", + "figtext", + "fill", + "fill_between", + "fill_betweenx", + "findobj", + "flag", + "gca", + "gcf", + "gci", + "get", + "get_backend", + "get_cmap", + "get_current_fig_manager", + "get_figlabels", + "get_fignums", + "get_plot_commands", + "get_scale_docs", + "get_scale_names", + "getp", + "ginput", + "gray", + "grid", + "hexbin", + "hist", + "hist2d", + "hlines", + "hot", + "hsv", + "importlib", + "imread", + "imsave", + "imshow", + "inferno", + "inspect", + "install_repl_d", + "matshow", + "minorticks_off", + "minorticks_on", + "mlab", + { "name": "figure", "reads": ["rcParams"] }, + "new_figure_manager", + "nipy_spectral", + "np", + "pause", + "pcolor", + "pcolormesh", + "phase_spectrum", + "pie", + "pink", + "plasma", + "plot", + "plot_date", + "plotfile", + "plotting", + "polar", + "prism", + "psd", + "pylab_setup", + "quiver", + "quiverkey", + "rc", + "rcParams", + "rcParamsDefault", + "rcParamsOrig", + "rc_context", + "rcdefaults", + "rcsetup", + "re", + "register_cmap", + "rgrids", + "savefig", + "sca", + "scatter", + "sci", + "semilogx", + "semilogy", + "set_cmap", + "setp", + "show", + "silent_list", + "specgram", + "spring", + "spy", + "stackplot", + "stem", + "step", + "streamplot", + "style", + "subplot", + { "name": "subplot_tool", "updates": ["rcParams"] }, + "subplot2grid", + "subplots", + "subplots_adjust", + "summer", + "suptitle", + { "name": "switch_backend", "updates": ["rcParams"] }, + "sys", + "table", + "text", + "thetagrids", + "tick_params", + "ticklabel_format", + "tight_layout", + "time", + "title", + "tricontour", + "tricontourf", + "tripcolor", + "triplot", + "twinx", + "twiny", + "uninstall_repl_displayhook", + "violinplot", + "viridis", + "vlines", + "waitforbuttonpress", + "warn_deprecated", + "warnings", + "winter", + "xcorr", + { "name": "xkcd", "reads": ["rcParams"] }, + "xlabel", + "xlim", + "xscale", + "xticks", + "ylabel", + "ylim", + "yscale", + "yticks" + ] + } + } + } +} diff --git a/types/@msrvida-python-program-analysis/specs/numpy.json b/types/@msrvida-python-program-analysis/specs/numpy.json new file mode 100644 index 000000000000..2c5b0e9f1823 --- /dev/null +++ b/types/@msrvida-python-program-analysis/specs/numpy.json @@ -0,0 +1,540 @@ +{ + "numpy": { + "modules": { + "random": { + "functions": [ + { "name": "seed", "updates": ["$global"] } + ] + } + }, + "functions": [ + { "name": "abs", "returns": "ndarray" }, + { "name": "absolute", "returns": "ndarray" }, + { "name": "add", "returns": "ndarray" }, + { "name": "add_docstring", "returns": "ndarray" }, + { "name": "add_newdoc", "returns": "ndarray" }, + { "name": "add_newdoc_ufunc", "returns": "ndarray" }, + { "name": "alen", "returns": "ndarray" }, + { "name": "all", "returns": "ndarray" }, + { "name": "allclose", "returns": "ndarray" }, + { "name": "alltrue", "returns": "ndarray" }, + { "name": "amax", "returns": "ndarray" }, + { "name": "amin", "returns": "ndarray" }, + { "name": "angle", "returns": "ndarray" }, + { "name": "any", "returns": "ndarray" }, + { "name": "append", "returns": "ndarray" }, + { "name": "apply_along_axis", "returns": "ndarray" }, + { "name": "apply_over_axes", "returns": "ndarray" }, + { "name": "arange", "returns": "ndarray" }, + { "name": "arccos", "returns": "ndarray" }, + { "name": "arccosh", "returns": "ndarray" }, + { "name": "arcsin", "returns": "ndarray" }, + { "name": "arcsinh", "returns": "ndarray" }, + { "name": "arctan", "returns": "ndarray" }, + { "name": "arctan2", "returns": "ndarray" }, + { "name": "arctanh", "returns": "ndarray" }, + { "name": "argmax", "returns": "ndarray" }, + { "name": "argmin", "returns": "ndarray" }, + { "name": "argpartition", "returns": "ndarray" }, + { "name": "argsort", "returns": "ndarray" }, + { "name": "argwhere", "returns": "ndarray" }, + { "name": "around", "returns": "ndarray" }, + { "name": "array", "returns": "ndarray" }, + { "name": "array2string", "returns": "ndarray" }, + { "name": "array_equal", "returns": "ndarray" }, + { "name": "array_equiv", "returns": "ndarray" }, + { "name": "array_repr", "returns": "ndarray" }, + { "name": "array_split", "returns": "ndarray" }, + { "name": "array_str", "returns": "ndarray" }, + { "name": "asanyarray", "returns": "ndarray" }, + { "name": "asarray", "returns": "ndarray" }, + { "name": "asarray_chkfinite", "returns": "ndarray" }, + { "name": "ascontiguousarray", "returns": "ndarray" }, + { "name": "asfarray", "returns": "ndarray" }, + { "name": "asfortranarray", "returns": "ndarray" }, + { "name": "asmatrix", "returns": "ndarray" }, + { "name": "asscalar", "returns": "ndarray" }, + { "name": "atleast_1d", "returns": "ndarray" }, + { "name": "atleast_2d", "returns": "ndarray" }, + { "name": "atleast_3d", "returns": "ndarray" }, + { "name": "average", "returns": "ndarray" }, + { "name": "bartlett", "returns": "ndarray" }, + { "name": "base_repr", "returns": "ndarray" }, + { "name": "binary_repr", "returns": "ndarray" }, + { "name": "bincount", "returns": "ndarray" }, + { "name": "bitwise_and", "returns": "ndarray" }, + { "name": "bitwise_not", "returns": "ndarray" }, + { "name": "bitwise_or", "returns": "ndarray" }, + { "name": "bitwise_xor", "returns": "ndarray" }, + { "name": "blackman", "returns": "ndarray" }, + { "name": "block", "returns": "ndarray" }, + { "name": "bmat", "returns": "ndarray" }, + { "name": "broadcast", "returns": "broadcast" }, + { "name": "broadcast_arrays", "returns": "broadcast" }, + { "name": "broadcast_to", "returns": "broadcast" }, + { "name": "busday_count", "returns": "ndarray" }, + { "name": "busday_offset", "returns": "ndarray" }, + "byte_bounds", + { "name": "can_cast", "returns": "ndarray" }, + { "name": "cbrt", "returns": "ndarray" }, + { "name": "cdouble", "returns": "ndarray" }, + { "name": "ceil", "returns": "ndarray" }, + { "name": "choose", "returns": "ndarray" }, + { "name": "clip", "returns": "ndarray" }, + { "name": "clongdouble", "returns": "ndarray" }, + { "name": "clongfloat", "returns": "ndarray" }, + { "name": "column_stack", "returns": "ndarray" }, + { "name": "common_type", "returns": "ndarray" }, + { "name": "compare_chararrays", "returns": "ndarray" }, + { "name": "compress", "returns": "ndarray" }, + { "name": "concatenate", "returns": "ndarray" }, + { "name": "conj", "returns": "ndarray" }, + { "name": "conjugate", "returns": "ndarray" }, + { "name": "convolve", "returns": "ndarray" }, + { "name": "copy", "returns": "ndarray" }, + { "name": "copysign", "returns": "ndarray" }, + { "name": "copyto", "returns": "ndarray" }, + { "name": "corrcoef", "returns": "ndarray" }, + { "name": "correlate", "returns": "ndarray" }, + { "name": "cos", "returns": "ndarray" }, + { "name": "cosh", "returns": "ndarray" }, + { "name": "count_nonzero", "returns": "ndarray" }, + { "name": "cov", "returns": "ndarray" }, + { "name": "cross", "returns": "ndarray" }, + { "name": "csingle", "returns": "ndarray" }, + { "name": "cumprod", "returns": "ndarray" }, + { "name": "cumproduct", "returns": "ndarray" }, + { "name": "cumsum", "returns": "ndarray" }, + { "name": "datetime64", "returns": "ndarray" }, + { "name": "datetime_as_string", "returns": "ndarray" }, + { "name": "datetime_data", "returns": "ndarray" }, + { "name": "deg2rad", "returns": "ndarray" }, + { "name": "degrees", "returns": "ndarray" }, + { "name": "delete", "returns": "ndarray" }, + { "name": "deprecate", "returns": "ndarray" }, + { "name": "deprecate_with_doc", "returns": "ndarray" }, + { "name": "diag", "returns": "ndarray" }, + { "name": "diag_indices", "returns": "ndarray" }, + { "name": "diag_indices_from", "returns": "ndarray" }, + { "name": "diagflat", "returns": "ndarray" }, + { "name": "diagonal", "returns": "ndarray" }, + { "name": "diff", "returns": "ndarray" }, + { "name": "digitize", "returns": "ndarray" }, + { "name": "disp", "returns": "ndarray" }, + { "name": "divide", "returns": "ndarray" }, + { "name": "divmod", "returns": "ndarray" }, + { "name": "dot", "returns": "ndarray" }, + { "name": "double", "returns": "ndarray" }, + { "name": "dsplit", "returns": "ndarray" }, + { "name": "dstack", "returns": "ndarray" }, + { "name": "dtype", "returns": "ndarray" }, + { "name": "ediff1d", "returns": "ndarray" }, + { "name": "einsum", "returns": "ndarray" }, + { "name": "einsum_path", "returns": "ndarray" }, + { "name": "empty", "returns": "ndarray" }, + { "name": "empty_like", "returns": "ndarray" }, + { "name": "equal", "returns": "ndarray" }, + { "name": "errstate", "returns": "ndarray" }, + { "name": "exp", "returns": "ndarray" }, + { "name": "exp2", "returns": "ndarray" }, + { "name": "expand_dims", "returns": "ndarray" }, + { "name": "expm1", "returns": "ndarray" }, + { "name": "extract", "returns": "ndarray" }, + { "name": "eye", "returns": "ndarray" }, + { "name": "fabs", "returns": "ndarray" }, + { "name": "fastCopyAndTranspose", "returns": "ndarray" }, + { "name": "fill_diagonal", "returns": "ndarray" }, + { "name": "find_common_type", "returns": "ndarray" }, + { "name": "finfo", "returns": "ndarray" }, + { "name": "fix", "returns": "ndarray" }, + { "name": "flatiter", "returns": "ndarray" }, + { "name": "flatnonzero", "returns": "ndarray" }, + { "name": "flexible", "returns": "ndarray" }, + { "name": "flip", "returns": "ndarray" }, + { "name": "fliplr", "returns": "ndarray" }, + { "name": "flipud", "returns": "ndarray" }, + { "name": "float", "returns": "ndarray" }, + { "name": "float16", "returns": "ndarray" }, + { "name": "float32", "returns": "ndarray" }, + { "name": "float64", "returns": "ndarray" }, + { "name": "float_", "returns": "ndarray" }, + { "name": "float_power", "returns": "ndarray" }, + { "name": "floating", "returns": "ndarray" }, + { "name": "floor", "returns": "ndarray" }, + { "name": "floor_divide", "returns": "ndarray" }, + { "name": "fmax", "returns": "ndarray" }, + { "name": "fmin", "returns": "ndarray" }, + { "name": "fmod", "returns": "ndarray" }, + { "name": "format_float_positional", "returns": "ndarray" }, + { "name": "format_float_scientific", "returns": "ndarray" }, + { "name": "format_parser", "returns": "ndarray" }, + { "name": "frexp", "returns": "ndarray" }, + { "name": "frombuffer", "returns": "ndarray" }, + { "name": "fromfile", "returns": "ndarray" }, + { "name": "fromfunction", "returns": "ndarray" }, + { "name": "fromiter", "returns": "ndarray" }, + { "name": "frompyfunc", "returns": "ndarray" }, + { "name": "fromregex", "returns": "ndarray" }, + { "name": "fromstring", "returns": "ndarray" }, + { "name": "full", "returns": "ndarray" }, + { "name": "full_like", "returns": "ndarray" }, + { "name": "fv", "returns": "ndarray" }, + { "name": "gcd", "returns": "ndarray" }, + { "name": "generic", "returns": "ndarray" }, + { "name": "genfromtxt", "returns": "ndarray" }, + { "name": "geomspace", "returns": "ndarray" }, + { "name": "get_array_wrap", "returns": "ndarray" }, + { "name": "get_include", "returns": "ndarray" }, + { "name": "get_printoptions", "returns": "ndarray" }, + { "name": "getbufsize", "returns": "ndarray" }, + { "name": "geterr", "returns": "ndarray" }, + { "name": "geterrcall", "returns": "ndarray" }, + { "name": "geterrobj", "returns": "ndarray" }, + { "name": "gradient", "returns": "ndarray" }, + { "name": "greater", "returns": "ndarray" }, + { "name": "greater_equal", "returns": "ndarray" }, + { "name": "half", "returns": "ndarray" }, + { "name": "hamming", "returns": "ndarray" }, + { "name": "hanning", "returns": "ndarray" }, + { "name": "heaviside", "returns": "ndarray" }, + { "name": "histogram", "returns": "ndarray" }, + { "name": "histogram2d", "returns": "ndarray" }, + { "name": "histogram_bin_edges", "returns": "ndarray" }, + { "name": "histogramdd", "returns": "ndarray" }, + { "name": "hsplit", "returns": "ndarray" }, + { "name": "hstack", "returns": "ndarray" }, + { "name": "hypot", "returns": "ndarray" }, + { "name": "i0", "returns": "ndarray" }, + { "name": "identity", "returns": "ndarray" }, + { "name": "iinfo", "returns": "ndarray" }, + { "name": "imag", "returns": "ndarray" }, + { "name": "in1d", "returns": "ndarray" }, + { "name": "indices", "returns": "ndarray" }, + { "name": "inexact", "returns": "ndarray" }, + { "name": "info", "returns": "ndarray" }, + { "name": "inner", "returns": "ndarray" }, + { "name": "insert", "returns": "ndarray" }, + { "name": "int_asbuffer", "returns": "ndarray" }, + { "name": "intc", "returns": "ndarray" }, + { "name": "integer", "returns": "ndarray" }, + { "name": "interp", "returns": "ndarray" }, + { "name": "intersect1d", "returns": "ndarray" }, + { "name": "intp", "returns": "ndarray" }, + { "name": "invert", "returns": "ndarray" }, + { "name": "ipmt", "returns": "ndarray" }, + { "name": "irr", "returns": "ndarray" }, + { "name": "is_busday", "returns": "ndarray" }, + { "name": "isclose", "returns": "ndarray" }, + { "name": "iscomplex", "returns": "ndarray" }, + { "name": "iscomplexobj", "returns": "ndarray" }, + { "name": "isfinite", "returns": "ndarray" }, + { "name": "isfortran", "returns": "ndarray" }, + { "name": "isin", "returns": "ndarray" }, + { "name": "isinf", "returns": "ndarray" }, + { "name": "isnan", "returns": "ndarray" }, + { "name": "isnat", "returns": "ndarray" }, + { "name": "isneginf", "returns": "ndarray" }, + { "name": "isposinf", "returns": "ndarray" }, + { "name": "isreal", "returns": "ndarray" }, + { "name": "isrealobj", "returns": "ndarray" }, + { "name": "isscalar", "returns": "ndarray" }, + { "name": "issctype", "returns": "ndarray" }, + { "name": "issubclass_", "returns": "ndarray" }, + { "name": "issubdtype", "returns": "ndarray" }, + { "name": "issubsctype", "returns": "ndarray" }, + { "name": "iterable", "returns": "ndarray" }, + { "name": "ix_", "returns": "ndarray" }, + { "name": "kaiser", "returns": "ndarray" }, + { "name": "kron", "returns": "ndarray" }, + { "name": "lcm", "returns": "ndarray" }, + { "name": "ldexp", "returns": "ndarray" }, + { "name": "left_shift", "returns": "ndarray" }, + { "name": "less", "returns": "ndarray" }, + { "name": "less_equal", "returns": "ndarray" }, + { "name": "lexsort", "returns": "ndarray" }, + { "name": "linspace", "returns": "ndarray" }, + { "name": "load", "returns": "ndarray" }, + { "name": "loads", "returns": "ndarray" }, + { "name": "loadtxt", "returns": "ndarray" }, + { "name": "log", "returns": "ndarray" }, + { "name": "log10", "returns": "ndarray" }, + { "name": "log1p", "returns": "ndarray" }, + { "name": "log2", "returns": "ndarray" }, + { "name": "logaddexp", "returns": "ndarray" }, + { "name": "logaddexp2", "returns": "ndarray" }, + { "name": "logical_and", "returns": "ndarray" }, + { "name": "logical_not", "returns": "ndarray" }, + { "name": "logical_or", "returns": "ndarray" }, + { "name": "logical_xor", "returns": "ndarray" }, + { "name": "logspace", "returns": "ndarray" }, + { "name": "long", "returns": "ndarray" }, + { "name": "longcomplex", "returns": "ndarray" }, + { "name": "longdouble", "returns": "ndarray" }, + { "name": "longfloat", "returns": "ndarray" }, + { "name": "longlong", "returns": "ndarray" }, + { "name": "lookfor", "returns": "ndarray" }, + { "name": "mafromtxt", "returns": "ndarray" }, + { "name": "mask_indices", "returns": "ndarray" }, + { "name": "mat", "returns": "ndarray" }, + { "name": "matmul", "returns": "ndarray" }, + { "name": "matrix", "returns": "ndarray" }, + { "name": "max", "returns": "ndarray" }, + { "name": "maximum", "returns": "ndarray" }, + { "name": "maximum_sctype", "returns": "ndarray" }, + "may_share_memory", + { "name": "mean", "returns": "ndarray" }, + { "name": "median", "returns": "ndarray" }, + { "name": "memmap", "returns": "ndarray" }, + { "name": "meshgrid", "returns": "ndarray" }, + { "name": "min", "returns": "ndarray" }, + { "name": "min_scalar_type", "returns": "ndarray" }, + { "name": "minimum", "returns": "ndarray" }, + { "name": "mintypecode", "returns": "ndarray" }, + { "name": "mirr", "returns": "ndarray" }, + { "name": "mod", "returns": "ndarray" }, + { "name": "modf", "returns": "ndarray" }, + { "name": "moveaxis", "returns": "ndarray" }, + { "name": "msort", "returns": "ndarray" }, + { "name": "multiply", "returns": "ndarray" }, + { "name": "nan_to_num", "returns": "ndarray" }, + { "name": "nanargmax", "returns": "ndarray" }, + { "name": "nanargmin", "returns": "ndarray" }, + { "name": "nancumprod", "returns": "ndarray" }, + { "name": "nancumsum", "returns": "ndarray" }, + { "name": "nanmax", "returns": "ndarray" }, + { "name": "nanmean", "returns": "ndarray" }, + { "name": "nanmedian", "returns": "ndarray" }, + { "name": "nanmin", "returns": "ndarray" }, + { "name": "nanpercentile", "returns": "ndarray" }, + { "name": "nanprod", "returns": "ndarray" }, + { "name": "nanquantile", "returns": "ndarray" }, + { "name": "nanstd", "returns": "ndarray" }, + { "name": "nansum", "returns": "ndarray" }, + { "name": "nanvar", "returns": "ndarray" }, + { "name": "ndarray", "returns": "ndarray" }, + { "name": "ndenumerate", "returns": "ndarray" }, + { "name": "ndfromtxt", "returns": "ndarray" }, + { "name": "ndim", "returns": "ndarray" }, + { "name": "negative", "returns": "ndarray" }, + { "name": "nested_iters", "returns": "ndarray" }, + { "name": "nextafter", "returns": "ndarray" }, + { "name": "nonzero", "returns": "ndarray" }, + { "name": "not_equal", "returns": "ndarray" }, + { "name": "nper", "returns": "ndarray" }, + { "name": "npv", "returns": "ndarray" }, + { "name": "obj2sctype", "returns": "ndarray" }, + { "name": "ones", "returns": "ndarray" }, + { "name": "ones_like", "returns": "ndarray" }, + { "name": "outer", "returns": "ndarray" }, + { "name": "packbits", "returns": "ndarray" }, + { "name": "pad", "returns": "ndarray" }, + { "name": "partition", "returns": "ndarray" }, + { "name": "percentile", "returns": "ndarray" }, + { "name": "piecewise", "returns": "ndarray" }, + { "name": "place", "returns": "ndarray" }, + { "name": "pmt", "returns": "ndarray" }, + { "name": "poly", "returns": "ndarray" }, + { "name": "polyadd", "returns": "ndarray" }, + { "name": "polyder", "returns": "ndarray" }, + { "name": "polydiv", "returns": "ndarray" }, + { "name": "polyfit", "returns": "ndarray" }, + { "name": "polyint", "returns": "ndarray" }, + { "name": "polymul", "returns": "ndarray" }, + { "name": "polysub", "returns": "ndarray" }, + { "name": "polyval", "returns": "ndarray" }, + { "name": "positive", "returns": "ndarray" }, + { "name": "power", "returns": "ndarray" }, + { "name": "ppmt", "returns": "ndarray" }, + { "name": "printoptions", "returns": "ndarray" }, + { "name": "prod", "returns": "ndarray" }, + { "name": "product", "returns": "ndarray" }, + { "name": "promote_types", "returns": "ndarray" }, + { "name": "ptp", "returns": "ndarray" }, + { "name": "put", "returns": "ndarray" }, + { "name": "put_along_axis", "returns": "ndarray" }, + { "name": "putmask", "returns": "ndarray" }, + { "name": "pv", "returns": "ndarray" }, + { "name": "quantile", "returns": "ndarray" }, + { "name": "rad2deg", "returns": "ndarray" }, + { "name": "radians", "returns": "ndarray" }, + { "name": "rank", "returns": "ndarray" }, + { "name": "rate", "returns": "ndarray" }, + { "name": "ravel", "returns": "ndarray" }, + { "name": "ravel_multi_index", "returns": "ndarray" }, + { "name": "real", "returns": "ndarray" }, + { "name": "real_if_close", "returns": "ndarray" }, + { "name": "recfromcsv", "returns": "ndarray" }, + { "name": "recfromtxt", "returns": "ndarray" }, + { "name": "reciprocal", "returns": "ndarray" }, + { "name": "remainder", "returns": "ndarray" }, + { "name": "repeat", "returns": "ndarray" }, + { "name": "require", "returns": "ndarray" }, + { "name": "reshape", "returns": "ndarray" }, + { "name": "resize", "returns": "ndarray" }, + { "name": "result_type", "returns": "ndarray" }, + { "name": "right_shift", "returns": "ndarray" }, + { "name": "rint", "returns": "ndarray" }, + { "name": "roll", "returns": "ndarray" }, + { "name": "rollaxis", "returns": "ndarray" }, + { "name": "roots", "returns": "ndarray" }, + { "name": "rot90", "returns": "ndarray" }, + { "name": "round", "returns": "ndarray" }, + { "name": "round_", "returns": "ndarray" }, + { "name": "row_stack", "returns": "ndarray" }, + "safe_eval", + "save", + "savetxt", + "savez", + "savez_compressed", + { "name": "sctype2char", "returns": "ndarray" }, + { "name": "searchsorted", "returns": "ndarray" }, + { "name": "select", "returns": "ndarray" }, + { "name": "set_numeric_ops", "returns": "ndarray" }, + { "name": "set_printoptions", "returns": "ndarray" }, + { "name": "set_string_function", "returns": "ndarray" }, + { "name": "setbufsize", "returns": "ndarray" }, + { "name": "setdiff1d", "returns": "ndarray" }, + { "name": "seterr", "returns": "ndarray" }, + { "name": "seterrcall", "returns": "ndarray" }, + { "name": "seterrobj", "returns": "ndarray" }, + { "name": "setxor1d", "returns": "ndarray" }, + { "name": "shape", "returns": "ndarray" }, + { "name": "shares_memory", "returns": "ndarray" }, + { "name": "short", "returns": "ndarray" }, + { "name": "show_config", "returns": "ndarray" }, + { "name": "sign", "returns": "ndarray" }, + { "name": "signbit", "returns": "ndarray" }, + { "name": "signedinteger", "returns": "ndarray" }, + { "name": "sin", "returns": "ndarray" }, + { "name": "sinc", "returns": "ndarray" }, + { "name": "single", "returns": "ndarray" }, + { "name": "singlecomplex", "returns": "ndarray" }, + { "name": "sinh", "returns": "ndarray" }, + "size", + { "name": "sometrue", "returns": "ndarray" }, + { "name": "sort", "returns": "ndarray" }, + { "name": "sort_complex", "returns": "ndarray" }, + { "name": "source", "returns": "ndarray" }, + { "name": "spacing", "returns": "ndarray" }, + { "name": "split", "returns": "ndarray" }, + { "name": "sqrt", "returns": "ndarray" }, + { "name": "square", "returns": "ndarray" }, + { "name": "squeeze", "returns": "ndarray" }, + { "name": "stack", "returns": "ndarray" }, + { "name": "std", "returns": "ndarray" }, + { "name": "subtract", "returns": "ndarray" }, + { "name": "sum", "returns": "ndarray" }, + { "name": "swapaxes", "returns": "ndarray" }, + { "name": "take", "returns": "ndarray" }, + { "name": "take_along_axis", "returns": "ndarray" }, + { "name": "tan", "returns": "ndarray" }, + { "name": "tanh", "returns": "ndarray" }, + { "name": "tensordot", "returns": "ndarray" }, + { "name": "tile", "returns": "ndarray" }, + { "name": "timedelta64", "returns": "ndarray" }, + { "name": "trace", "returns": "ndarray" }, + { "name": "transpose", "returns": "ndarray" }, + { "name": "trapz", "returns": "ndarray" }, + { "name": "tri", "returns": "ndarray" }, + { "name": "tril", "returns": "ndarray" }, + { "name": "tril_indices", "returns": "ndarray" }, + { "name": "tril_indices_from", "returns": "ndarray" }, + { "name": "trim_zeros", "returns": "ndarray" }, + { "name": "triu", "returns": "ndarray" }, + { "name": "triu_indices", "returns": "ndarray" }, + { "name": "triu_indices_from", "returns": "ndarray" }, + { "name": "true_divide", "returns": "ndarray" }, + { "name": "trunc", "returns": "ndarray" }, + "typename", + { "name": "union1d", "returns": "ndarray" }, + { "name": "unique", "returns": "ndarray" }, + { "name": "unpackbits", "returns": "ndarray" }, + { "name": "unravel_index", "returns": "ndarray" }, + { "name": "unwrap", "returns": "ndarray" }, + { "name": "vander", "returns": "ndarray" }, + { "name": "var", "returns": "ndarray" }, + { "name": "vdot", "returns": "ndarray" }, + { "name": "vectorize", "returns": "ndarray" }, + { "name": "vsplit", "returns": "ndarray" }, + { "name": "vstack", "returns": "ndarray" }, + { "name": "where", "returns": "ndarray" }, + { "name": "who", "returns": "ndarray" }, + { "name": "zeros", "returns": "ndarray" }, + { "name": "zeros_like", "returns": "ndarray" } + ], + "types": { + "ndarray": { + "methods": [ + { "name": "all", "returns": "ndarray" }, + { "name": "any", "returns": "ndarray" }, + { "name": "argmax", "returns": "ndarray" }, + { "name": "argmin", "returns": "ndarray" }, + { "name": "argpartition", "returns": "ndarray" }, + { "name": "argsort", "returns": "ndarray" }, + { "name": "astype", "returns": "ndarray" }, + { "name": "base", "returns": "ndarray" }, + { "name": "byteswap", "returns": "ndarray", "updates": [0] }, + { "name": "choose", "returns": "ndarray" }, + { "name": "clip", "returns": "ndarray" }, + { "name": "compress", "returns": "ndarray" }, + { "name": "conj", "returns": "ndarray" }, + { "name": "conjugate", "returns": "ndarray" }, + { "name": "copy", "returns": "ndarray" }, + { "name": "cumprod", "returns": "ndarray" }, + { "name": "cumsum", "returns": "ndarray" }, + { "name": "data", "returns": "ndarray" }, + { "name": "diagonal", "returns": "ndarray" }, + { "name": "dot", "returns": "ndarray" }, + { "name": "dump", "returns": "ndarray" }, + { "name": "dumps", "returns": "ndarray" }, + { "name": "fill", "returns": "ndarray", "updates": [0] }, + { "name": "flags", "returns": "ndarray" }, + { "name": "flat", "returns": "ndarray" }, + { "name": "flatten", "returns": "ndarray" }, + { "name": "getfield", "returns": "ndarray" }, + { "name": "imag", "returns": "ndarray" }, + { "name": "item", "returns": "ndarray" }, + { "name": "itemset", "returns": "ndarray" }, + { "name": "itemsize", "returns": "ndarray" }, + { "name": "max", "returns": "ndarray" }, + { "name": "mean", "returns": "ndarray" }, + { "name": "min", "returns": "ndarray" }, + { "name": "nbytes", "returns": "ndarray" }, + { "name": "ndim", "returns": "ndarray" }, + { "name": "newbyteorder", "returns": "ndarray" }, + { "name": "nonzero", "returns": "ndarray" }, + { "name": "partition", "returns": "ndarray" }, + { "name": "prod", "returns": "ndarray" }, + { "name": "ptp", "returns": "ndarray" }, + { "name": "put", "returns": "ndarray" }, + { "name": "ravel", "returns": "ndarray" }, + { "name": "real", "returns": "ndarray" }, + { "name": "repeat", "returns": "ndarray" }, + { "name": "reshape", "returns": "ndarray" }, + { "name": "resize", "returns": "ndarray", "updates": [0] }, + { "name": "round", "returns": "ndarray" }, + { "name": "searchsorted", "returns": "ndarray" }, + { "name": "setfield", "returns": "ndarray" }, + { "name": "setflags", "returns": "ndarray" }, + { "name": "shape", "returns": "ndarray" }, + { "name": "size", "returns": "ndarray" }, + { "name": "sort", "updates": [0] }, + { "name": "squeeze", "returns": "ndarray", "updates": [0] }, + { "name": "std", "returns": "ndarray" }, + { "name": "strides", "returns": "ndarray" }, + { "name": "sum", "returns": "ndarray" }, + { "name": "swapaxes", "returns": "ndarray" }, + { "name": "take", "returns": "ndarray" }, + { "name": "tobytes", "returns": "ndarray" }, + { "name": "tofile", "returns": "ndarray" }, + { "name": "tolist", "returns": "ndarray" }, + { "name": "tostring", "returns": "ndarray" }, + { "name": "trace", "returns": "ndarray" }, + { "name": "transpose", "returns": "ndarray" }, + { "name": "var", "returns": "ndarray" }, + { "name": "view", "returns": "ndarray" } + ] + } + } + } +} diff --git a/types/@msrvida-python-program-analysis/specs/pandas.json b/types/@msrvida-python-program-analysis/specs/pandas.json new file mode 100644 index 000000000000..3e72991a2e07 --- /dev/null +++ b/types/@msrvida-python-program-analysis/specs/pandas.json @@ -0,0 +1,276 @@ +{ + "pandas": { + "functions": [ + "array", + "bdate_range", + "concat", + "crosstab", + "cut", + "date_range", + "datetime", + "describe_option", + "eval", + "factorize", + "get_dummies", + "get_option", + "infer_freq", + "interval_range", + "isna", + "isnull", + "lreshape", + "melt", + "merge", + "merge_asof", + "merge_ordered", + "notna", + "notnull", + "option_context", + "period_range", + "pivot", + "pivot_table", + "qcut", + { "name": "read_clipboard", "returns": "DataFrame" }, + { "name": "read_csv", "returns": "DataFrame" }, + { "name": "read_excel", "returns": "DataFrame" }, + { "name": "read_feather", "returns": "DataFrame" }, + { "name": "read_fwf", "returns": "DataFrame" }, + { "name": "read_gbq", "returns": "DataFrame" }, + { "name": "read_hdf", "returns": "DataFrame" }, + { "name": "read_html", "returns": "DataFrame" }, + { "name": "read_json", "returns": "DataFrame" }, + { "name": "read_msgpack", "returns": "DataFrame" }, + { "name": "read_parquet", "returns": "DataFrame" }, + { "name": "read_pickle", "returns": "DataFrame" }, + { "name": "read_sas", "returns": "DataFrame" }, + { "name": "read_sql", "returns": "DataFrame" }, + { "name": "read_sql_query", "returns": "DataFrame" }, + { "name": "read_sql_table", "returns": "DataFrame" }, + { "name": "read_stata", "returns": "DataFrame" }, + { "name": "read_table", "returns": "DataFrame" }, + "reset_option", + "set_eng_float_format", + "set_option", + "show_versions", + "test", + "timedelta_range", + "to_datetime", + "to_msgpack", + "to_numeric", + "to_pickle", + "to_timedelta", + "unique", + "value_counts", + "wide_to_long" + ], + "types": { + "DataFrame": { + "methods": [ + "abs", + "add", + "add_prefix", + "add_suffix", + "agg", + "aggregate", + "align", + "all", + "any", + "append", + "apply", + "applymap", + "as_blocks", + "as_matrix", + "asfreq", + "asof", + "assign", + "astype", + "at_time", + "between_time", + "bfill", + "bool", + "boxplot", + "clip", + "clip_lower", + "clip_upper", + "combine", + "combine_first", + "compound", + "convert_objects", + "copy", + "corr", + "corrwith", + "count", + "cov", + "cummax", + "cummin", + "cumprod", + "cumsum", + "describe", + "diff", + "div", + "divide", + "dot", + "drop", + "drop_duplicates", + "droplevel", + "dropna", + "duplicated", + "eq", + "equals", + "eval", + "ewm", + "expanding", + { "name": "ffill", "updates": [0] }, + { "name": "fillna", "updates": [0] }, + "filter", + "first", + "first_valid_index", + "floordiv", + "from_csv", + "from_dict", + "from_items", + "from_records", + "ge", + "get", + "get_dtype_counts", + "get_ftype_counts", + "get_value", + "get_values", + "groupby", + "gt", + "head", + "hist", + "idxmax", + "idxmin", + "infer_objects", + "info", + "insert", + "interpolate", + "isin", + "isna", + "isnull", + "items", + "iteritems", + "iterrows", + "itertuples", + "join", + "keys", + "kurt", + "kurtosis", + "last", + "last_valid_index", + "le", + "lookup", + "lt", + "mad", + "mask", + "max", + "mean", + "median", + { "name": "melt", "updates": [0] }, + "memory_usage", + "merge", + "min", + "mod", + "mode", + "mul", + "multiply", + "ne", + "nlargest", + "notna", + "notnull", + "nsmallest", + "nunique", + "pct_change", + "pipe", + "pivot", + "pivot_table", + "plot", + { "name": "pop", "updates": [0] }, + "pow", + "prod", + "product", + "quantile", + "query", + "radd", + "rank", + "rdiv", + "reindex", + "reindex_axis", + "reindex_like", + "rename", + "rename_axis", + "reorder_levels", + "replace", + "resample", + "reset_index", + "rfloordiv", + "rmod", + "rmul", + "rolling", + "round", + "rpow", + "rsub", + "rtruediv", + "sample", + "select", + "select_dtypes", + "sem", + "set_axis", + "set_index", + "set_value", + "shift", + "skew", + "slice_shift", + "sort_index", + "sort_values", + "squeeze", + "stack", + "std", + "sub", + "subtract", + "sum", + "swapaxes", + "swaplevel", + "tail", + "take", + "to_clipboard", + "to_csv", + "to_dense", + "to_dict", + "to_excel", + "to_feather", + "to_gbq", + "to_hdf", + "to_html", + "to_json", + "to_latex", + "to_msgpack", + "to_numpy", + "to_panel", + "to_parquet", + "to_period", + "to_pickle", + "to_records", + "to_sparse", + "to_sql", + "to_stata", + "to_string", + "to_timestamp", + "to_xarray", + "transform", + "transpose", + "truediv", + "truncate", + "tshift", + "tz_convert", + "tz_localize", + "unstack", + { "name": "update", "updates": [0] }, + "var", + "where", + "xs" + ] + } + } + } +} diff --git a/types/@msrvida-python-program-analysis/specs/random.json b/types/@msrvida-python-program-analysis/specs/random.json new file mode 100644 index 000000000000..5b02a9f217c6 --- /dev/null +++ b/types/@msrvida-python-program-analysis/specs/random.json @@ -0,0 +1,28 @@ +{ + "random": { + "functions": [ + "betavariate", + "choice", + "choices", + "expovariate", + "gammavariate", + "gauss", + "getrandbits", + "getstate", + "lognormvariate", + "normalvariate", + "paretovariate", + "randint", + "random", + "randrange", + "sample", + { "name": "seed", "updates": ["$global"] }, + { "name": "setstate", "updates": ["$global"] }, + "shuffle", + "triangular", + "uniform", + "vonmisesvariate", + "weibullvariate" + ] + } +} diff --git a/types/@msrvida-python-program-analysis/specs/sklearn.json b/types/@msrvida-python-program-analysis/specs/sklearn.json new file mode 100644 index 000000000000..bce8507fc478 --- /dev/null +++ b/types/@msrvida-python-program-analysis/specs/sklearn.json @@ -0,0 +1,13811 @@ +{ + "sklearn": { + "modules": { + "cluster": { + "modules": { + "tests": { + "modules": { + "common": { + "functions": ["generate_clustered_data"], + "types": {} + }, + "test_affinity_propagation": { + "functions": [ + "test_affinity_propagation", + "test_affinity_propagation_predict", + "test_affinity_propagation_predict_error", + "test_affinity_propagation_fit_non_convergence", + "test_affinity_propagation_equal_mutual_similarities", + "test_affinity_propagation_predict_non_convergence", + "test_equal_similarities_and_preferences" + ], + "types": {} + }, + "test_bicluster": { + "functions": [ + "test_get_submatrix", + "_test_shape_indices", + "test_spectral_coclustering", + "test_spectral_biclustering", + "_do_scale_test", + "_do_bistochastic_test", + "test_scale_normalize", + "test_bistochastic_normalize", + "test_log_normalize", + "test_fit_best_piecewise", + "test_project_and_cluster", + "test_perfect_checkerboard", + "test_errors" + ], + "types": { + "MockBiclustering": { + "methods": [ + { + "name": "__init__", + "updates": [], + "returns": "MockBiclustering" + }, + "__init__", + "get_indices" + ] + } + } + }, + "test_birch": { + "functions": [ + "test_n_samples_leaves_roots", + "test_partial_fit", + "test_birch_predict", + "test_n_clusters", + "test_sparse_X", + "check_branching_factor", + "test_branching_factor", + "check_threshold", + "test_threshold" + ], + "types": {} + }, + "test_dbscan": { + "functions": [ + "test_dbscan_similarity", + "test_dbscan_feature", + "test_dbscan_sparse", + "test_dbscan_no_core_samples", + "test_dbscan_callable", + "test_dbscan_metric_params", + "test_dbscan_balltree", + "test_input_validation", + "test_dbscan_badargs", + "test_pickle", + "test_boundaries", + "test_weighted_dbscan", + "test_dbscan_precomputed_metric_with_degenerate_input_arrays", + "test_dbscan_precomputed_metric_with_initial_rows_zero" + ], + "types": {} + }, + "test_feature_agglomeration": { + "functions": ["test_feature_agglomeration"], + "types": {} + }, + "test_hierarchical": { + "functions": [ + "test_linkage_misc", + "test_structured_linkage_tree", + "test_unstructured_linkage_tree", + "test_height_linkage_tree", + "test_agglomerative_clustering_wrong_arg_memory", + "test_ward_agglomeration", + "assess_same_labelling", + "test_scikit_vs_scipy", + "test_connectivity_propagation", + "test_ward_tree_children_order", + "test_ward_linkage_tree_return_distance", + "test_connectivity_fixing_non_lil", + "test_int_float_dict", + "test_connectivity_callable", + "test_connectivity_ignores_diagonal", + "test_compute_full_tree", + "test_n_components", + "test_agg_n_clusters", + "test_affinity_passed_to_fix_connectivity", + "test_small_distance_threshold", + "test_cluster_distances_with_distance_threshold", + "test_dist_threshold_invalid_parameters", + "test_n_components_deprecation" + ], + "types": { + "FakeAffinity": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "FakeAffinity" + }, + { + "name": "increment", + "updates": [0] + } + ] + } + } + }, + "test_k_means": { + "functions": [ + "test_labels_assignment_and_inertia", + "test_minibatch_update_consistency", + "_check_fitted_model", + "test_k_means_new_centers", + "test_k_means_precompute_distances_flag", + "test_k_means_plus_plus_init_not_precomputed", + "test_k_means_random_init_not_precomputed", + "test_k_means_n_init", + "test_k_means_fortran_aligned_data", + "test_mb_kmeans_verbose", + "test_minibatch_init_with_large_k", + "test_minibatch_k_means_init_multiple_runs_with_explicit_centers", + "test_minibatch_sensible_reassign_fit", + "test_minibatch_sensible_reassign_partial_fit", + "test_minibatch_reassign", + "test_minibatch_with_many_reassignments", + "test_sparse_mb_k_means_callable_init", + "test_mini_batch_k_means_random_init_partial_fit", + "test_minibatch_default_init_size", + "test_minibatch_tol", + "test_minibatch_set_init_size", + "test_k_means_copyx", + "test_k_means_non_collapsed", + "test_int_input", + "test_transform", + "test_fit_transform", + "test_full_vs_elkan", + "test_n_init", + "test_k_means_function", + "test_x_squared_norms_init_centroids", + "test_max_iter_error", + "test_k_means_init_centers", + "test_sparse_validate_centers", + "test_less_centers_than_unique_points", + "_sort_centers", + "test_weighted_vs_repeated", + "test_unit_weights_vs_no_weights", + "test_scaled_weights", + "test_sample_weight_length", + "test_check_sample_weight", + "test_iter_attribute", + "test_k_means_empty_cluster_relocated" + ], + "types": {} + }, + "test_mean_shift": { + "functions": [ + "test_estimate_bandwidth", + "test_estimate_bandwidth_1sample", + "test_mean_shift_negative_bandwidth", + "test_estimate_bandwidth_with_sparse_matrix", + "test_parallel", + "test_meanshift_predict", + "test_meanshift_all_orphans", + "test_unfitted", + "test_cluster_intensity_tie", + "test_bin_seeds" + ], + "types": {} + }, + "test_optics": { + "functions": [ + "test_extract_xi", + "test_cluster_hierarchy_", + "test_correct_number_of_clusters", + "test_minimum_number_of_sample_check", + "test_bad_extract", + "test_bad_reachability", + "test_close_extract", + "test_min_samples_edge_case", + "test_min_cluster_size_invalid2", + "test_processing_order", + "test_compare_to_ELKI", + "test_wrong_cluster_method", + "test_extract_dbscan", + "test_precomputed_dists" + ], + "types": {} + }, + "test_spectral": { + "functions": [ + "test_spectral_unknown_mode", + "test_spectral_unknown_assign_labels", + "test_spectral_clustering_sparse", + "test_affinities", + "test_spectral_clustering_with_arpack_amg_solvers" + ], + "types": {} + } + }, + "types": {}, + "functions": [] + }, + "affinity_propagation_": { + "functions": [ + "_equal_similarities_and_preferences", + "affinity_propagation" + ], + "types": { + "AffinityPropagation": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "AffinityPropagation" + }, + { + "name": "fit", + "updates": [0] + }, + "predict" + ] + } + } + }, + "bicluster": { + "functions": [ + "_scale_normalize", + "_bistochastic_normalize", + "_log_normalize" + ], + "types": { + "BaseSpectral": { + "methods": ["_check_parameters", "fit", "_svd", "_k_means"] + }, + "SpectralCoclustering": { + "methods": [ + { + "name": "__init__", + "updates": [], + "returns": "SpectralCoclustering" + }, + "__init__", + { + "name": "_fit", + "updates": [0] + }, + "fit" + ] + }, + "SpectralBiclustering": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "SpectralBiclustering" + }, + "_check_parameters", + { + "name": "_fit", + "updates": [0] + }, + "_fit_best_piecewise", + "_project_and_cluster", + "fit" + ] + } + } + }, + "birch": { + "functions": ["_iterate_sparse_X", "_split_node"], + "types": { + "_CFNode": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "_CFNode" + }, + { + "name": "append_subcluster", + "updates": [0] + }, + { + "name": "update_split_subclusters", + "updates": [0] + }, + { + "name": "insert_cf_subcluster", + "updates": [0] + } + ] + }, + "_CFSubcluster": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "_CFSubcluster" + }, + { + "name": "update", + "updates": [0] + }, + "merge_subcluster" + ] + }, + "Birch": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "Birch" + }, + { + "name": "fit", + "updates": [0] + }, + { + "name": "_fit", + "updates": [0] + }, + "_get_leaves", + { + "name": "partial_fit", + "updates": [0] + }, + "_check_fit", + "predict", + "transform", + { + "name": "_global_clustering", + "updates": [0] + } + ] + } + } + }, + "dbscan_": { + "functions": ["dbscan"], + "types": { + "DBSCAN": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "DBSCAN" + }, + { + "name": "fit", + "updates": [0] + }, + { + "name": "fit_predict", + "updates": [0] + } + ] + } + } + }, + "hierarchical": { + "functions": [ + { + "name": "_fix_connectivity", + "updates": [1] + }, + "_single_linkage_tree", + "ward_tree", + { + "name": "linkage_tree", + "updates": [1] + }, + "_complete_linkage", + "_average_linkage", + "_single_linkage", + "_hc_cut" + ], + "types": { + "AgglomerativeClustering": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "AgglomerativeClustering" + }, + { + "name": "fit", + "updates": [0] + } + ] + }, + "FeatureAgglomeration": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "FeatureAgglomeration" + }, + { + "name": "fit", + "updates": [0] + } + ] + } + } + }, + "k_means_": { + "functions": [ + "_k_init", + "_validate_center_shape", + "_tolerance", + "_check_sample_weight", + "k_means", + "_kmeans_single_elkan", + "_kmeans_single_lloyd", + { + "name": "_labels_inertia_precompute_dense", + "updates": [4] + }, + "_labels_inertia", + "_init_centroids", + { + "name": "_mini_batch_step", + "updates": [3, 4, 5] + }, + { + "name": "_mini_batch_convergence", + "updates": [7] + } + ], + "types": { + "KMeans": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "KMeans" + }, + "_check_test_data", + { + "name": "fit", + "updates": [0] + }, + { + "name": "fit_predict", + "updates": [0] + }, + { + "name": "fit_transform", + "updates": [0] + }, + "transform", + "_transform", + "predict", + "score" + ] + }, + "MiniBatchKMeans": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "MiniBatchKMeans" + }, + { + "name": "fit", + "updates": [0] + }, + "_labels_inertia_minibatch", + { + "name": "partial_fit", + "updates": [0] + }, + "predict", + { + "name": "fit_predict", + "updates": [0] + }, + { + "name": "fit_transform", + "updates": [0] + }, + "transform", + "score" + ] + } + } + }, + "mean_shift_": { + "functions": [ + "estimate_bandwidth", + "_mean_shift_single_seed", + "mean_shift", + "get_bin_seeds" + ], + "types": { + "MeanShift": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "MeanShift" + }, + { + "name": "fit", + "updates": [0] + }, + "predict" + ] + } + } + }, + "optics_": { + "functions": [ + "_validate_size", + "_compute_core_distances_", + "compute_optics_graph", + { + "name": "_set_reach_dist", + "updates": [1, 2] + }, + "cluster_optics_dbscan", + "cluster_optics_xi", + "_extend_region", + "_update_filter_sdas", + "_correct_predecessor", + "_xi_cluster", + "_extract_xi_labels" + ], + "types": { + "OPTICS": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "OPTICS" + }, + { + "name": "fit", + "updates": [0] + } + ] + } + } + }, + "setup": { + "functions": ["configuration"], + "types": {} + }, + "spectral": { + "functions": [ + { + "name": "discretize", + "updates": [0] + }, + "spectral_clustering" + ], + "types": { + "SpectralClustering": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "SpectralClustering" + }, + { + "name": "fit", + "updates": [0] + } + ] + } + } + } + }, + "types": { + "SpectralClustering": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "SpectralClustering" + }, + { + "name": "fit", + "updates": [0] + } + ] + }, + "MeanShift": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "MeanShift" + }, + { + "name": "fit", + "updates": [0] + }, + "predict" + ] + }, + "AffinityPropagation": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "AffinityPropagation" + }, + { + "name": "fit", + "updates": [0] + }, + "predict" + ] + }, + "AgglomerativeClustering": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "AgglomerativeClustering" + }, + { + "name": "fit", + "updates": [0] + } + ] + }, + "FeatureAgglomeration": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "FeatureAgglomeration" + }, + { + "name": "fit", + "updates": [0] + } + ] + }, + "KMeans": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "KMeans" + }, + "_check_test_data", + { + "name": "fit", + "updates": [0] + }, + { + "name": "fit_predict", + "updates": [0] + }, + { + "name": "fit_transform", + "updates": [0] + }, + "transform", + "_transform", + "predict", + "score" + ] + }, + "MiniBatchKMeans": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "MiniBatchKMeans" + }, + { + "name": "fit", + "updates": [0] + }, + "_labels_inertia_minibatch", + { + "name": "partial_fit", + "updates": [0] + }, + "predict", + { + "name": "fit_predict", + "updates": [0] + }, + { + "name": "fit_transform", + "updates": [0] + }, + "transform", + "score" + ] + }, + "DBSCAN": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "DBSCAN" + }, + { + "name": "fit", + "updates": [0] + }, + { + "name": "fit_predict", + "updates": [0] + } + ] + }, + "OPTICS": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "OPTICS" + }, + { + "name": "fit", + "updates": [0] + } + ] + }, + "SpectralBiclustering": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "SpectralBiclustering" + }, + "_check_parameters", + { + "name": "_fit", + "updates": [0] + }, + "_fit_best_piecewise", + "_project_and_cluster", + "fit" + ] + }, + "SpectralCoclustering": { + "methods": [ + { + "name": "__init__", + "updates": [], + "returns": "SpectralCoclustering" + }, + "__init__", + { + "name": "_fit", + "updates": [0] + }, + "fit" + ] + }, + "Birch": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "Birch" + }, + { + "name": "fit", + "updates": [0] + }, + { + "name": "_fit", + "updates": [0] + }, + "_get_leaves", + { + "name": "partial_fit", + "updates": [0] + }, + "_check_fit", + "predict", + "transform", + { + "name": "_global_clustering", + "updates": [0] + } + ] + } + }, + "functions": [ + "spectral_clustering", + "mean_shift", + "estimate_bandwidth", + "get_bin_seeds", + "affinity_propagation", + "ward_tree", + { + "name": "linkage_tree", + "updates": [1] + }, + "k_means", + "dbscan", + "cluster_optics_dbscan", + "compute_optics_graph", + "cluster_optics_xi" + ] + }, + "compose": { + "modules": { + "tests": { + "modules": { + "test_column_transformer": { + "functions": [ + "test_column_transformer", + "test_column_transformer_dataframe", + "test_column_transformer_sparse_array", + "test_column_transformer_list", + "test_column_transformer_sparse_stacking", + "test_column_transformer_mixed_cols_sparse", + "test_column_transformer_sparse_threshold", + "test_column_transformer_error_msg_1D", + "test_2D_transformer_output", + "test_2D_transformer_output_pandas", + "test_column_transformer_invalid_transformer", + "test_make_column_transformer", + "test_make_column_transformer_pandas", + "test_make_column_transformer_kwargs", + "test_make_column_transformer_remainder_transformer", + "test_column_transformer_get_set_params", + "test_column_transformer_named_estimators", + "test_column_transformer_cloning", + "test_column_transformer_get_feature_names", + "test_column_transformer_special_strings", + "test_column_transformer_remainder", + "test_column_transformer_no_remaining_remainder_transformer", + "test_column_transformer_drops_all_remainder_transformer", + "test_column_transformer_sparse_remainder_transformer", + "test_column_transformer_drop_all_sparse_remainder_transformer", + "test_column_transformer_get_set_params_with_remainder", + "test_column_transformer_no_estimators", + "test_column_transformer_no_estimators_set_params", + "test_column_transformer_callable_specifier", + "test_column_transformer_negative_column_indexes" + ], + "types": { + "Trans": { + "methods": ["fit", "transform"] + }, + "DoubleTrans": { + "methods": ["fit", "transform"] + }, + "SparseMatrixTrans": { + "methods": ["fit", "transform"] + }, + "TransNo2D": { + "methods": ["fit", "transform"] + }, + "TransRaise": { + "methods": ["fit", "transform"] + }, + "TransAssert": { + "methods": ["fit", "transform"] + }, + "NoTrans": { + "methods": ["fit", "predict"] + } + } + }, + "test_target": { + "functions": [ + "test_transform_target_regressor_error", + "test_transform_target_regressor_invertible", + "_check_standard_scaled", + "_check_shifted_by_one", + "test_transform_target_regressor_functions", + "test_transform_target_regressor_functions_multioutput", + "test_transform_target_regressor_2d_transformer_multioutput", + "test_transform_target_regressor_multi_to_single", + "test_transform_target_regressor_ensure_y_array" + ], + "types": { + "DummyCheckerArrayTransformer": { + "methods": ["fit", "transform", "inverse_transform"] + }, + "DummyCheckerListRegressor": { + "methods": ["fit", "predict"] + }, + "DummyTransformer": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "DummyTransformer" + }, + { + "name": "fit", + "updates": [0] + }, + "transform", + "inverse_transform" + ] + } + } + } + }, + "types": {}, + "functions": [] + } + }, + "types": {}, + "functions": [] + }, + "covariance": { + "modules": { + "tests": { + "modules": { + "test_covariance": { + "functions": [ + "test_covariance", + "test_shrunk_covariance", + "test_ledoit_wolf", + "_naive_ledoit_wolf_shrinkage", + "test_ledoit_wolf_small", + "test_ledoit_wolf_large", + "test_oas" + ], + "types": {} + }, + "test_elliptic_envelope": { + "functions": [ + "test_elliptic_envelope", + "test_score_samples", + "test_raw_values_deprecation", + "test_threshold_deprecation" + ], + "types": {} + }, + "test_graphical_lasso": { + "functions": [ + "test_graphical_lasso", + "test_graphical_lasso_iris", + "test_graph_lasso_2D", + "test_graphical_lasso_iris_singular" + ], + "types": {} + }, + "test_graph_lasso": { + "functions": [], + "types": {} + }, + "test_robust_covariance": { + "functions": [ + "test_mcd", + "test_fast_mcd_on_invalid_input", + "test_mcd_class_on_invalid_input", + "launch_mcd_on_dataset", + "test_mcd_issue1127", + "test_mcd_issue3367", + "test_mcd_support_covariance_is_zero", + "test_mcd_increasing_det_warning" + ], + "types": {} + } + }, + "types": {}, + "functions": [] + }, + "elliptic_envelope": { + "functions": [], + "types": { + "EllipticEnvelope": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "EllipticEnvelope" + }, + { + "name": "fit", + "updates": [0] + }, + "decision_function", + "score_samples", + "predict", + "score" + ] + } + } + }, + "empirical_covariance_": { + "functions": ["log_likelihood", "empirical_covariance"], + "types": { + "EmpiricalCovariance": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "EmpiricalCovariance" + }, + { + "name": "_set_covariance", + "updates": [0] + }, + "get_precision", + { + "name": "fit", + "updates": [0] + }, + "score", + "error_norm", + "mahalanobis" + ] + } + } + }, + "graph_lasso_": { + "functions": [ + "_objective", + "_dual_gap", + "alpha_max", + "graphical_lasso", + "graphical_lasso_path" + ], + "types": { + "GraphicalLasso": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "GraphicalLasso" + }, + { + "name": "fit", + "updates": [0] + } + ] + }, + "GraphicalLassoCV": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "GraphicalLassoCV" + }, + { + "name": "fit", + "updates": [0] + } + ] + }, + "GraphLasso": { + "methods": [ + { + "name": "fit", + "updates": [0] + } + ] + }, + "GraphLassoCV": { + "methods": [ + { + "name": "fit", + "updates": [0] + } + ] + } + } + }, + "robust_covariance": { + "functions": ["c_step", "_c_step", "select_candidates", "fast_mcd"], + "types": { + "MinCovDet": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "MinCovDet" + }, + { + "name": "fit", + "updates": [0] + }, + { + "name": "correct_covariance", + "updates": [0] + }, + { + "name": "reweight_covariance", + "updates": [0] + } + ] + } + } + }, + "shrunk_covariance_": { + "functions": [ + "shrunk_covariance", + "ledoit_wolf_shrinkage", + "ledoit_wolf", + "oas" + ], + "types": { + "ShrunkCovariance": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "ShrunkCovariance" + }, + { + "name": "fit", + "updates": [0] + } + ] + }, + "LedoitWolf": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "LedoitWolf" + }, + { + "name": "fit", + "updates": [0] + } + ] + }, + "OAS": { + "methods": [ + { + "name": "fit", + "updates": [0] + } + ] + } + } + } + }, + "types": { + "EmpiricalCovariance": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "EmpiricalCovariance" + }, + { + "name": "_set_covariance", + "updates": [0] + }, + "get_precision", + { + "name": "fit", + "updates": [0] + }, + "score", + "error_norm", + "mahalanobis" + ] + }, + "ShrunkCovariance": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "ShrunkCovariance" + }, + { + "name": "fit", + "updates": [0] + } + ] + }, + "LedoitWolf": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "LedoitWolf" + }, + { + "name": "fit", + "updates": [0] + } + ] + }, + "OAS": { + "methods": [ + { + "name": "fit", + "updates": [0] + } + ] + }, + "MinCovDet": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "MinCovDet" + }, + { + "name": "fit", + "updates": [0] + }, + { + "name": "correct_covariance", + "updates": [0] + }, + { + "name": "reweight_covariance", + "updates": [0] + } + ] + }, + "GraphLasso": { + "methods": [ + { + "name": "fit", + "updates": [0] + } + ] + }, + "GraphLassoCV": { + "methods": [ + { + "name": "fit", + "updates": [0] + } + ] + }, + "GraphicalLasso": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "GraphicalLasso" + }, + { + "name": "fit", + "updates": [0] + } + ] + }, + "GraphicalLassoCV": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "GraphicalLassoCV" + }, + { + "name": "fit", + "updates": [0] + } + ] + }, + "EllipticEnvelope": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "EllipticEnvelope" + }, + { + "name": "fit", + "updates": [0] + }, + "decision_function", + "score_samples", + "predict", + "score" + ] + } + }, + "functions": [ + "empirical_covariance", + "log_likelihood", + "shrunk_covariance", + "ledoit_wolf", + "ledoit_wolf_shrinkage", + "oas", + "fast_mcd", + "graphical_lasso" + ] + }, + "cross_decomposition": { + "modules": { + "tests": { + "modules": { + "test_pls": { + "functions": [ + "test_pls", + "test_convergence_fail", + "test_PLSSVD", + "test_univariate_pls_regression", + "test_predict_transform_copy", + "test_scale_and_stability", + "test_pls_errors" + ], + "types": {} + } + }, + "types": {}, + "functions": [] + }, + "cca_": { + "functions": [], + "types": { + "CCA": { + "methods": [ + { + "name": "__init__", + "updates": [], + "returns": "CCA" + }, + "__init__" + ] + } + } + }, + "pls_": { + "functions": [ + "_nipals_twoblocks_inner_loop", + "_svd_cross_product", + "_center_scale_xy" + ], + "types": { + "_PLS": { + "methods": [ + { + "name": "fit", + "updates": [0] + }, + "transform", + "predict", + { + "name": "fit_transform", + "updates": [0] + }, + "_more_tags" + ] + }, + "PLSRegression": { + "methods": [ + { + "name": "__init__", + "updates": [], + "returns": "PLSRegression" + }, + "__init__", + { + "name": "fit", + "updates": [0] + }, + "transform", + "predict", + { + "name": "fit_transform", + "updates": [0] + } + ] + }, + "PLSCanonical": { + "methods": [ + { + "name": "__init__", + "updates": [], + "returns": "PLSCanonical" + }, + "__init__", + { + "name": "fit", + "updates": [0] + }, + "transform", + "predict", + { + "name": "fit_transform", + "updates": [0] + } + ] + }, + "PLSSVD": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "PLSSVD" + }, + { + "name": "fit", + "updates": [0] + }, + "transform", + { + "name": "fit_transform", + "updates": [0] + } + ] + } + } + } + }, + "types": {}, + "functions": [] + }, + "cross_validation": { + "functions": [ + { "name": "train_test_split", "returns": "List[numpy.ndarray]" } + ] + }, + "datasets": { + "modules": { + "data": { + "modules": {}, + "types": {}, + "functions": [] + }, + "descr": { + "modules": {}, + "types": {}, + "functions": [] + }, + "images": { + "modules": {}, + "types": {}, + "functions": [] + }, + "tests": { + "modules": { + "data": { + "modules": { + "openml": { + "modules": { + "1": { + "modules": {}, + "types": {}, + "functions": [] + }, + "2": { + "modules": {}, + "types": {}, + "functions": [] + }, + "3": { + "modules": {}, + "types": {}, + "functions": [] + }, + "61": { + "modules": {}, + "types": {}, + "functions": [] + }, + "292": { + "modules": {}, + "types": {}, + "functions": [] + }, + "561": { + "modules": {}, + "types": {}, + "functions": [] + }, + "1119": { + "modules": {}, + "types": {}, + "functions": [] + }, + "40589": { + "modules": {}, + "types": {}, + "functions": [] + }, + "40675": { + "modules": {}, + "types": {}, + "functions": [] + }, + "40945": { + "modules": {}, + "types": {}, + "functions": [] + }, + "40966": { + "modules": {}, + "types": {}, + "functions": [] + } + }, + "types": {}, + "functions": [] + } + }, + "types": {}, + "functions": [] + }, + "test_20news": { + "functions": [ + "test_20news", + "test_20news_length_consistency", + "test_20news_vectorized" + ], + "types": {} + }, + "test_base": { + "functions": [ + "_remove_dir", + "test_data_home", + "test_default_empty_load_files", + "test_default_load_files", + "test_load_files_w_categories_desc_and_encoding", + "test_load_files_wo_load_content", + "test_load_sample_images", + "test_load_digits", + "test_load_digits_n_class_lt_10", + "test_load_sample_image", + "test_load_missing_sample_image_error", + "test_load_diabetes", + "test_load_linnerud", + "test_load_iris", + "test_load_wine", + "test_load_breast_cancer", + "test_load_boston", + "test_loads_dumps_bunch", + "test_bunch_pickle_generated_with_0_16_and_read_with_0_17", + "test_bunch_dir", + "test_refresh_cache" + ], + "types": {} + }, + "test_california_housing": { + "functions": ["fetch", "test_fetch"], + "types": {} + }, + "test_common": { + "functions": ["check_return_X_y"], + "types": {} + }, + "test_covtype": { + "functions": ["fetch", "test_fetch"], + "types": {} + }, + "test_kddcup99": { + "functions": ["test_percent10", "test_shuffle"], + "types": {} + }, + "test_lfw": { + "functions": [ + "setup_module", + "teardown_module", + "test_load_empty_lfw_people", + "test_load_fake_lfw_people", + "test_load_fake_lfw_people_too_restrictive", + "test_load_empty_lfw_pairs", + "test_load_fake_lfw_pairs" + ], + "types": {} + }, + "test_mldata": { + "functions": [], + "types": {} + }, + "test_openml": { + "functions": [ + "_test_features_list", + "_fetch_dataset_from_openml", + "_monkey_patch_webbased_functions", + "test_decode_iris", + "test_decode_anneal", + "test_decode_cpu", + "test_decode_emotions", + "test_retry_with_clean_cache", + "test_retry_with_clean_cache_http_error", + "test_fetch_openml_raises_illegal_argument" + ], + "types": { + "MockHTTPResponse": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "MockHTTPResponse" + }, + "read", + "tell", + "seek", + "close", + "info" + ] + } + } + }, + "test_rcv1": { + "functions": ["test_fetch_rcv1"], + "types": {} + }, + "test_samples_generator": { + "functions": [ + "test_make_classification", + "test_make_classification_informative_features", + "test_make_multilabel_classification_return_sequences", + "test_make_multilabel_classification_return_indicator", + "test_make_multilabel_classification_return_indicator_sparse", + "test_make_hastie_10_2", + "test_make_regression", + "test_make_regression_multitarget", + "test_make_blobs", + "test_make_blobs_n_samples_list", + "test_make_blobs_n_samples_list_with_centers", + "test_make_blobs_error", + "test_make_friedman1", + "test_make_friedman2", + "test_make_friedman3", + "test_make_low_rank_matrix", + "test_make_sparse_coded_signal", + "test_make_sparse_uncorrelated", + "test_make_spd_matrix", + "test_make_swiss_roll", + "test_make_s_curve", + "test_make_biclusters", + "test_make_checkerboard", + "test_make_moons", + "test_make_circles" + ], + "types": {} + }, + "test_svmlight_format": { + "functions": [ + "test_load_svmlight_file", + "test_load_svmlight_file_fd", + "test_load_svmlight_file_multilabel", + "test_load_svmlight_files", + "test_load_svmlight_file_n_features", + "test_load_compressed", + "test_load_invalid_file", + "test_load_invalid_order_file", + "test_load_zero_based", + "test_load_zero_based_auto", + "test_load_with_qid", + "test_load_invalid_file2", + "test_not_a_filename", + "test_invalid_filename", + "test_dump", + "test_dump_multilabel", + "test_dump_concise", + "test_dump_comment", + "test_dump_invalid", + "test_dump_query_id", + "test_load_with_long_qid", + "test_load_zeros", + "test_load_offset_exhaustive_splits", + "test_load_with_offsets_error" + ], + "types": {} + } + }, + "types": {}, + "functions": [] + }, + "base": { + "functions": [ + "get_data_home", + "clear_data_home", + "load_files", + "load_data", + "load_wine", + "load_iris", + "load_breast_cancer", + "load_digits", + "load_diabetes", + "load_linnerud", + "load_boston", + "load_sample_images", + "load_sample_image", + "_pkl_filepath", + "_sha256", + "_fetch_remote", + "_refresh_cache" + ], + "types": {} + }, + "california_housing": { + "functions": ["fetch_california_housing"], + "types": {} + }, + "covtype": { + "functions": ["fetch_covtype"], + "types": {} + }, + "kddcup99": { + "functions": ["fetch_kddcup99", "_fetch_brute_kddcup99", "_mkdirp"], + "types": {} + }, + "lfw": { + "functions": [ + "_check_fetch_lfw", + "_load_imgs", + "_fetch_lfw_people", + "fetch_lfw_people", + "_fetch_lfw_pairs", + "fetch_lfw_pairs" + ], + "types": {} + }, + "mldata": { + "functions": ["setup_module", "teardown_module"], + "types": {} + }, + "olivetti_faces": { + "functions": ["fetch_olivetti_faces"], + "types": {} + }, + "openml": { + "functions": [ + "_get_local_path", + "_retry_with_clean_cache", + "_open_openml_url", + "_get_json_content_from_openml_api", + "_split_sparse_columns", + "_sparse_data_to_array", + "_convert_arff_data", + "_get_data_info_by_name", + "_get_data_description_by_id", + "_get_data_features", + "_get_data_qualities", + "_get_data_shape", + "_download_data_arff", + "_verify_target_data_type", + "_valid_data_column_names", + "fetch_openml" + ], + "types": {} + }, + "rcv1": { + "functions": [ + "fetch_rcv1", + "_inverse_permutation", + "_find_permutation" + ], + "types": {} + }, + "samples_generator": { + "functions": [ + "_generate_hypercube", + { + "name": "make_classification", + "updates": [7] + }, + "make_multilabel_classification", + "make_hastie_10_2", + "make_regression", + "make_circles", + "make_moons", + "make_blobs", + "make_friedman1", + "make_friedman2", + "make_friedman3", + "make_low_rank_matrix", + "make_sparse_coded_signal", + "make_sparse_uncorrelated", + "make_spd_matrix", + "make_sparse_spd_matrix", + "make_swiss_roll", + "make_s_curve", + "make_gaussian_quantiles", + "_shuffle", + "make_biclusters", + "make_checkerboard" + ], + "types": {} + }, + "setup": { + "functions": ["configuration"], + "types": {} + }, + "species_distributions": { + "functions": [ + "_load_coverage", + "_load_csv", + "construct_grids", + "fetch_species_distributions" + ], + "types": {} + }, + "svmlight_format": { + "functions": [ + "load_svmlight_file", + "_gen_open", + "_open_and_load", + "load_svmlight_files", + "_dump_svmlight", + "dump_svmlight_file" + ], + "types": {} + }, + "twenty_newsgroups": { + "functions": [ + "_download_20newsgroups", + "strip_newsgroup_header", + "strip_newsgroup_quoting", + "strip_newsgroup_footer", + "fetch_20newsgroups", + "fetch_20newsgroups_vectorized" + ], + "types": {} + } + }, + "types": {}, + "functions": [ + "load_breast_cancer", + "load_boston", + "load_diabetes", + "load_digits", + "load_files", + "load_iris", + "load_linnerud", + "load_sample_images", + "load_sample_image", + "load_wine", + "get_data_home", + "clear_data_home", + "fetch_covtype", + "fetch_kddcup99", + "fetch_lfw_pairs", + "fetch_lfw_people", + "fetch_20newsgroups", + "fetch_20newsgroups_vectorized", + "fetch_openml", + { + "name": "make_classification", + "updates": [7] + }, + "make_multilabel_classification", + "make_hastie_10_2", + "make_regression", + "make_blobs", + "make_moons", + "make_circles", + "make_friedman1", + "make_friedman2", + "make_friedman3", + "make_low_rank_matrix", + "make_sparse_coded_signal", + "make_sparse_uncorrelated", + "make_spd_matrix", + "make_swiss_roll", + "make_s_curve", + "make_sparse_spd_matrix", + "make_gaussian_quantiles", + "make_biclusters", + "make_checkerboard", + "load_svmlight_file", + "load_svmlight_files", + "dump_svmlight_file", + "fetch_olivetti_faces", + "fetch_species_distributions", + "fetch_california_housing", + "fetch_rcv1" + ] + }, + "decomposition": { + "modules": { + "tests": { + "modules": { + "test_dict_learning": { + "functions": [ + "test_sparse_encode_shapes_omp", + "test_dict_learning_shapes", + "test_dict_learning_overcomplete", + "test_dict_learning_reconstruction", + "test_dict_learning_reconstruction_parallel", + "test_dict_learning_lassocd_readonly_data", + "test_dict_learning_nonzero_coefs", + "test_dict_learning_unknown_fit_algorithm", + "test_dict_learning_split", + "test_dict_learning_online_shapes", + "test_dict_learning_online_verbosity", + "test_dict_learning_online_estimator_shapes", + "test_dict_learning_online_overcomplete", + "test_dict_learning_online_initialization", + "test_dict_learning_online_readonly_initialization", + "test_dict_learning_online_partial_fit", + "test_sparse_encode_shapes", + "test_sparse_encode_input", + "test_sparse_encode_error", + "test_sparse_encode_error_default_sparsity", + "test_unknown_method", + "test_sparse_coder_estimator", + "test_sparse_coder_parallel_mmap" + ], + "types": {} + }, + "test_factor_analysis": { + "functions": [], + "types": {} + }, + "test_fastica": { + "functions": [ + "center_and_norm", + "test_gs", + "test_fastica_nowhiten", + "test_fastica_convergence_fail", + "test_non_square_fastica", + "test_fit_transform", + "test_inverse_transform", + "test_fastica_errors" + ], + "types": {} + }, + "test_incremental_pca": { + "functions": [ + "test_incremental_pca", + "test_incremental_pca_check_projection", + "test_incremental_pca_inverse", + "test_incremental_pca_validation", + "test_n_components_none", + "test_incremental_pca_set_params", + "test_incremental_pca_num_features_change", + "test_incremental_pca_batch_signs", + "test_incremental_pca_batch_values", + "test_incremental_pca_batch_rank", + "test_incremental_pca_partial_fit", + "test_incremental_pca_against_pca_iris", + "test_incremental_pca_against_pca_random_data", + "test_explained_variances", + "test_singular_values", + "test_whitening", + "test_incremental_pca_partial_fit_float_division" + ], + "types": {} + }, + "test_kernel_pca": { + "functions": [ + "test_kernel_pca", + "test_kernel_pca_invalid_parameters", + "test_kernel_pca_consistent_transform", + "test_kernel_pca_deterministic_output", + "test_kernel_pca_sparse", + "test_kernel_pca_linear_kernel", + "test_kernel_pca_n_components", + "test_remove_zero_eig", + "test_leave_zero_eig", + "test_kernel_pca_precomputed", + "test_kernel_pca_invalid_kernel" + ], + "types": {} + }, + "test_nmf": { + "functions": [ + "test_initialize_nn_output", + "test_parameter_checking", + "test_initialize_close", + "test_initialize_variants", + "test_nmf_transform_custom_init", + "test_n_components_greater_n_features", + "test_nmf_sparse_input", + "test_nmf_sparse_transform", + "test_non_negative_factorization_consistency", + "test_non_negative_factorization_checking", + "_beta_divergence_dense", + "test_beta_divergence", + "test_special_sparse_dot", + "test_nmf_negative_beta_loss", + "test_nmf_regularization", + "test_nmf_underflow" + ], + "types": {} + }, + "test_online_lda": { + "functions": [ + "_build_sparse_mtx", + "test_lda_default_prior_params", + "test_lda_fit_batch", + "test_lda_fit_online", + "test_lda_partial_fit", + "test_lda_dense_input", + "test_lda_transform", + "test_lda_partial_fit_dim_mismatch", + "test_invalid_params", + "test_lda_negative_input", + "test_lda_no_component_error", + "test_lda_transform_mismatch", + "test_lda_preplexity_mismatch", + "test_perplexity_input_format", + "test_lda_score_perplexity", + "test_lda_fit_perplexity", + "test_lda_empty_docs", + "test_dirichlet_expectation", + "check_verbosity" + ], + "types": {} + }, + "test_pca": { + "functions": [ + "test_pca", + "test_pca_arpack_solver", + "test_pca_randomized_solver", + "test_no_empty_slice_warning", + "test_whitening", + "test_singular_values", + "test_pca_check_projection", + "test_pca_inverse", + "test_randomized_pca_check_projection", + "test_randomized_pca_check_list", + "test_randomized_pca_inverse", + "test_n_components_mle", + "test_pca_dim", + "test_infer_dim_1", + "test_infer_dim_2", + "test_infer_dim_3", + "test_infer_dim_by_explained_variance", + "test_pca_score", + "test_pca_score2", + "test_pca_score3", + "test_pca_score_with_different_solvers", + "test_pca_zero_noise_variance_edge_cases", + "test_svd_solver_auto", + "test_pca_bad_solver", + "test_pca_deterministic_output", + "check_pca_float_dtype_preservation", + "check_pca_int_dtype_upcast_to_double" + ], + "types": {} + }, + "test_sparse_pca": { + "functions": [ + "generate_toy_data", + "test_scaling_fit_transform", + "test_pca_vs_spca" + ], + "types": {} + }, + "test_truncated_svd": { + "functions": [ + "test_algorithms", + "test_attributes", + "test_integers", + "test_explained_variance", + "test_singular_values", + "test_truncated_svd_eq_pca" + ], + "types": {} + } + }, + "types": {}, + "functions": [] + }, + "base": { + "functions": [], + "types": { + "_BasePCA": { + "methods": [ + "get_covariance", + "get_precision", + "transform", + "inverse_transform" + ] + } + } + }, + "dict_learning": { + "functions": [ + "_sparse_encode", + "sparse_encode", + { + "name": "_update_dict", + "updates": [0, 2] + }, + "dict_learning", + "dict_learning_online" + ], + "types": { + "SparseCodingMixin": { + "methods": [ + { + "name": "_set_sparse_coding_params", + "updates": [0] + }, + "transform" + ] + }, + "SparseCoder": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "SparseCoder" + }, + "fit", + "transform" + ] + }, + "DictionaryLearning": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "DictionaryLearning" + }, + { + "name": "fit", + "updates": [0] + }, + "transform" + ] + }, + "MiniBatchDictionaryLearning": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "MiniBatchDictionaryLearning" + }, + { + "name": "fit", + "updates": [0] + }, + { + "name": "partial_fit", + "updates": [0] + }, + "transform" + ] + } + } + }, + "factor_analysis": { + "functions": [], + "types": { + "FactorAnalysis": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "FactorAnalysis" + }, + { + "name": "fit", + "updates": [0] + }, + "transform", + "get_covariance", + "get_precision", + "score_samples", + "score" + ] + } + } + }, + "fastica_": { + "functions": [ + "_gs_decorrelation", + "_sym_decorrelation", + "_ica_def", + "_ica_par", + "_logcosh", + "_exp", + "_cube", + "fastica" + ], + "types": { + "FastICA": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "FastICA" + }, + { + "name": "_fit", + "updates": [0] + }, + { + "name": "fit_transform", + "updates": [0] + }, + { + "name": "fit", + "updates": [0] + }, + "transform", + "inverse_transform" + ] + } + } + }, + "incremental_pca": { + "functions": [], + "types": { + "IncrementalPCA": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "IncrementalPCA" + }, + { + "name": "fit", + "updates": [0] + }, + { + "name": "partial_fit", + "updates": [0] + } + ] + } + } + }, + "kernel_pca": { + "functions": [], + "types": { + "KernelPCA": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "KernelPCA" + }, + "_get_kernel", + { + "name": "_fit_transform", + "updates": [0] + }, + { + "name": "_fit_inverse_transform", + "updates": [0] + }, + { + "name": "fit", + "updates": [0] + }, + { + "name": "fit_transform", + "updates": [0] + }, + "transform", + "inverse_transform" + ] + } + } + }, + "nmf": { + "functions": [ + "norm", + "trace_dot", + "_check_init", + "_beta_divergence", + "_special_sparse_dot", + "_compute_regularization", + "_check_string_param", + "_beta_loss_to_float", + "_initialize_nmf", + "_update_coordinate_descent", + "_fit_coordinate_descent", + "_multiplicative_update_w", + "_multiplicative_update_h", + { + "name": "_fit_multiplicative_update", + "updates": [1, 2] + }, + "non_negative_factorization" + ], + "types": { + "NMF": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "NMF" + }, + { + "name": "fit_transform", + "updates": [0] + }, + { + "name": "fit", + "updates": [0] + }, + "transform", + "inverse_transform" + ] + } + } + }, + "online_lda": { + "functions": ["_update_doc_distribution"], + "types": { + "LatentDirichletAllocation": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "LatentDirichletAllocation" + }, + "_check_params", + { + "name": "_init_latent_vars", + "updates": [0] + }, + "_e_step", + { + "name": "_em_step", + "updates": [0] + }, + "_check_non_neg_array", + { + "name": "partial_fit", + "updates": [0] + }, + { + "name": "fit", + "updates": [0] + }, + "_unnormalized_transform", + "transform", + "_approx_bound", + "score", + "_perplexity_precomp_distr", + "perplexity" + ] + } + } + }, + "pca": { + "functions": ["_assess_dimension_", "_infer_dimension_"], + "types": { + "PCA": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "PCA" + }, + { + "name": "fit", + "updates": [0] + }, + { + "name": "fit_transform", + "updates": [0] + }, + { + "name": "_fit", + "updates": [0] + }, + { + "name": "_fit_full", + "updates": [0] + }, + { + "name": "_fit_truncated", + "updates": [0] + }, + "score_samples", + "score" + ] + } + } + }, + "setup": { + "functions": ["configuration"], + "types": {} + }, + "sparse_pca": { + "functions": [], + "types": { + "SparsePCA": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "SparsePCA" + }, + { + "name": "fit", + "updates": [0] + }, + "transform" + ] + }, + "MiniBatchSparsePCA": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "MiniBatchSparsePCA" + }, + { + "name": "fit", + "updates": [0] + }, + "transform" + ] + } + } + }, + "truncated_svd": { + "functions": [], + "types": { + "TruncatedSVD": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "TruncatedSVD" + }, + { + "name": "fit", + "updates": [0] + }, + { + "name": "fit_transform", + "updates": [0] + }, + "transform", + "inverse_transform" + ] + } + } + } + }, + "types": { + "NMF": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "NMF" + }, + { + "name": "fit_transform", + "updates": [0] + }, + { + "name": "fit", + "updates": [0] + }, + "transform", + "inverse_transform" + ] + }, + "PCA": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "PCA" + }, + { + "name": "fit", + "updates": [0] + }, + { + "name": "fit_transform", + "updates": [0] + }, + { + "name": "_fit", + "updates": [0] + }, + { + "name": "_fit_full", + "updates": [0] + }, + { + "name": "_fit_truncated", + "updates": [0] + }, + "score_samples", + "score" + ] + }, + "IncrementalPCA": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "IncrementalPCA" + }, + { + "name": "fit", + "updates": [0] + }, + { + "name": "partial_fit", + "updates": [0] + } + ] + }, + "KernelPCA": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "KernelPCA" + }, + "_get_kernel", + { + "name": "_fit_transform", + "updates": [0] + }, + { + "name": "_fit_inverse_transform", + "updates": [0] + }, + { + "name": "fit", + "updates": [0] + }, + { + "name": "fit_transform", + "updates": [0] + }, + "transform", + "inverse_transform" + ] + }, + "SparsePCA": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "SparsePCA" + }, + { + "name": "fit", + "updates": [0] + }, + "transform" + ] + }, + "MiniBatchSparsePCA": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "MiniBatchSparsePCA" + }, + { + "name": "fit", + "updates": [0] + }, + "transform" + ] + }, + "TruncatedSVD": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "TruncatedSVD" + }, + { + "name": "fit", + "updates": [0] + }, + { + "name": "fit_transform", + "updates": [0] + }, + "transform", + "inverse_transform" + ] + }, + "FastICA": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "FastICA" + }, + { + "name": "_fit", + "updates": [0] + }, + { + "name": "fit_transform", + "updates": [0] + }, + { + "name": "fit", + "updates": [0] + }, + "transform", + "inverse_transform" + ] + }, + "DictionaryLearning": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "DictionaryLearning" + }, + { + "name": "fit", + "updates": [0] + }, + "transform" + ] + }, + "MiniBatchDictionaryLearning": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "MiniBatchDictionaryLearning" + }, + { + "name": "fit", + "updates": [0] + }, + { + "name": "partial_fit", + "updates": [0] + }, + "transform" + ] + }, + "SparseCoder": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "SparseCoder" + }, + "fit", + "transform" + ] + }, + "FactorAnalysis": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "FactorAnalysis" + }, + { + "name": "fit", + "updates": [0] + }, + "transform", + "get_covariance", + "get_precision", + "score_samples", + "score" + ] + }, + "LatentDirichletAllocation": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "LatentDirichletAllocation" + }, + "_check_params", + { + "name": "_init_latent_vars", + "updates": [0] + }, + "_e_step", + { + "name": "_em_step", + "updates": [0] + }, + "_check_non_neg_array", + { + "name": "partial_fit", + "updates": [0] + }, + { + "name": "fit", + "updates": [0] + }, + "_unnormalized_transform", + "transform", + "_approx_bound", + "score", + "_perplexity_precomp_distr", + "perplexity" + ] + } + }, + "functions": [ + "non_negative_factorization", + "fastica", + "dict_learning", + "dict_learning_online", + "sparse_encode" + ] + }, + "ensemble": { + "modules": { + "tests": { + "modules": { + "test_bagging": { + "functions": [ + "test_classification", + "test_sparse_classification", + "test_regression", + "test_sparse_regression", + "test_bootstrap_samples", + "test_bootstrap_features", + "test_oob_score_classification", + "test_oob_score_regression", + "test_single_estimator", + "test_error", + "test_parallel_classification", + "test_parallel_regression", + "test_base_estimator", + "test_bagging_with_pipeline", + "test_bagging_sample_weight_unsupported_but_passed", + "test_warm_start", + "test_warm_start_smaller_n_estimators", + "test_warm_start_equal_n_estimators", + "test_warm_start_equivalence", + "test_warm_start_with_oob_score_fails", + "test_oob_score_removed_on_warm_start", + "test_oob_score_consistency", + "test_max_samples_consistency", + "test_set_oob_score_label_encoding", + { + "name": "replace", + "updates": [0] + }, + "test_bagging_regressor_with_missing_inputs", + "test_bagging_classifier_with_missing_inputs" + ], + "types": { + "CustomSVC": { + "methods": [ + { + "name": "fit", + "updates": [0] + } + ] + }, + "CustomSVR": { + "methods": [ + { + "name": "fit", + "updates": [0] + } + ] + }, + "DummySizeEstimator": { + "methods": [ + { + "name": "fit", + "updates": [0] + } + ] + }, + "DummyZeroEstimator": { + "methods": [ + { + "name": "fit", + "updates": [0] + }, + "predict" + ] + } + } + }, + "test_base": { + "functions": [ + "test_base", + "test_base_zero_n_estimators", + "test_base_not_int_n_estimators", + "test_set_random_states" + ], + "types": { + "AlphaParamPipeline": { + "methods": ["get_params"] + }, + "RevParamPipeline": { + "methods": ["get_params"] + } + } + }, + "test_forest": { + "functions": [ + "check_classification_toy", + "check_iris_criterion", + "check_boston_criterion", + "check_regressor_attributes", + "check_probability", + "check_importances", + "test_importances_asymptotic", + "check_unfitted_feature_importances", + "check_oob_score", + "check_oob_score_raise_error", + "check_gridsearch", + "check_parallel", + "check_pickle", + "check_multioutput", + "check_classes_shape", + "test_random_trees_dense_type", + "test_random_trees_dense_equal", + "test_random_hasher_sparse_data", + "test_parallel_train", + "test_distribution", + "check_max_leaf_nodes_max_depth", + "check_min_samples_split", + "check_min_samples_leaf", + "check_min_weight_fraction_leaf", + "check_sparse_input", + "check_memory_layout", + "check_class_weights", + "check_class_weight_balanced_and_bootstrap_multi_output", + "check_class_weight_errors", + "check_warm_start", + "check_warm_start_clear", + "check_warm_start_smaller_n_estimators", + "check_warm_start_equal_n_estimators", + "check_warm_start_oob", + "check_decision_path", + "test_forest_feature_importances_sum", + "test_forest_degenerate_feature_importances" + ], + "types": { + "MyBackend": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "MyBackend" + }, + { + "name": "start_call", + "updates": [0] + } + ] + } + } + }, + "test_gradient_boosting": { + "functions": [ + "check_classification_toy", + "test_classifier_parameter_checks", + "test_regressor_parameter_checks", + "test_loss_function", + "check_classification_synthetic", + "check_boston", + "check_iris", + "test_regression_synthetic", + "test_feature_importances", + "test_probability_log", + "test_check_inputs", + "test_check_inputs_predict", + "test_check_inputs_predict_stages", + "test_check_max_features", + "test_max_feature_regression", + "test_max_feature_auto", + "test_staged_predict", + "test_staged_predict_proba", + "test_serialization", + "test_degenerate_targets", + "test_quantile_loss", + "test_symbol_labels", + "test_float_class_labels", + "test_shape_y", + "test_mem_layout", + "test_oob_improvement", + "test_oob_improvement_raise", + "test_oob_multilcass_iris", + "test_verbose_output", + "test_more_verbose_output", + "early_stopping_monitor", + "test_complete_classification", + "test_complete_regression", + "test_zero_estimator_reg", + "test_zero_estimator_clf", + "test_warm_start_wo_nestimators_change", + "test_probability_exponential", + "test_non_uniform_weights_toy_edge_case_reg", + "test_non_uniform_weights_toy_edge_case_clf", + "check_sparse_input", + "test_gradient_boosting_early_stopping", + "test_gradient_boosting_validation_fraction", + "test_early_stopping_stratified", + "_make_multiclass", + "test_gradient_boosting_with_init_pipeline", + "test_early_stopping_n_classes", + "test_gbr_degenerate_feature_importances" + ], + "types": { + "_NoSampleWeightWrapper": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "_NoSampleWeightWrapper" + }, + "fit", + "predict", + "predict_proba" + ] + } + } + }, + "test_gradient_boosting_loss_functions": { + "functions": [ + "test_binomial_deviance", + "test_sample_weight_smoke", + "test_sample_weight_init_estimators", + "test_weighted_percentile", + "test_weighted_percentile_equal", + "test_weighted_percentile_zero_weight", + "test_quantile_loss_function", + "test_sample_weight_deviance", + "test_init_raw_predictions_shapes", + "test_init_raw_predictions_values" + ], + "types": {} + }, + "test_iforest": { + "functions": ["test_iforest_average_path_length"], + "types": {} + }, + "test_partial_dependence": { + "functions": [], + "types": {} + }, + "test_voting": { + "functions": [ + "test_weights_regressor", + "test_multilabel", + "test_sample_weight_kwargs" + ], + "types": { + "ClassifierErrorFit": { + "methods": ["fit"] + }, + "MockClassifier": { + "methods": ["fit"] + } + } + }, + "test_weight_boosting": { + "functions": [ + "test_samme_proba", + "test_oneclass_adaboost_proba", + "test_classification_toy", + "test_regression_toy", + "test_iris", + "test_boston", + "test_staged_predict", + "test_pickle", + "test_importances", + "test_error", + "test_sample_weight_missing", + "test_sparse_classification", + "test_sparse_regression", + "test_sample_weight_adaboost_regressor", + "test_multidimensional_X" + ], + "types": { + "MockEstimator": { + "methods": ["predict_proba"] + }, + "CustomSVC": { + "methods": [ + { + "name": "fit", + "updates": [0] + } + ] + }, + "CustomSVR": { + "methods": [ + { + "name": "fit", + "updates": [0] + } + ] + }, + "DummyEstimator": { + "methods": ["fit", "predict"] + } + } + } + }, + "types": {}, + "functions": [] + }, + "bagging": { + "functions": [ + "_generate_indices", + "_generate_bagging_indices", + "_parallel_build_estimators", + "_parallel_predict_proba", + "_parallel_predict_log_proba", + "_parallel_decision_function", + "_parallel_predict_regression" + ], + "types": { + "BaseBagging": { + "methods": [ + { + "name": "fit", + "updates": [0] + }, + "_parallel_args", + { + "name": "_fit", + "updates": [0] + }, + "_validate_y", + "_get_estimators_indices" + ] + }, + "BaggingClassifier": { + "methods": [ + { + "name": "__init__", + "updates": [], + "returns": "BaggingClassifier" + }, + "__init__", + "_validate_estimator", + { + "name": "_set_oob_score", + "updates": [0] + }, + { + "name": "_validate_y", + "updates": [0] + }, + "predict", + "predict_proba", + "predict_log_proba", + { + "name": "fit", + "updates": [0] + } + ] + }, + "BaggingRegressor": { + "methods": [ + { + "name": "__init__", + "updates": [], + "returns": "BaggingRegressor" + }, + "__init__", + "predict", + "_validate_estimator", + { + "name": "_set_oob_score", + "updates": [0] + }, + { + "name": "fit", + "updates": [0] + } + ] + } + } + }, + "base": { + "functions": ["_set_random_states", "_partition_estimators"], + "types": { + "BaseEnsemble": { + "methods": [ + { + "name": "_validate_estimator", + "updates": [0] + }, + "_make_estimator", + "__len__", + "__getitem__", + "__iter__" + ] + } + } + }, + "forest": { + "functions": [ + "_generate_sample_indices", + "_generate_unsampled_indices", + "_parallel_build_trees", + { + "name": "_accumulate_prediction", + "updates": [2] + } + ], + "types": { + "BaseForest": { + "methods": [ + "apply", + "decision_path", + { + "name": "fit", + "updates": [0] + }, + "_validate_y_class_weight", + "_validate_X_predict" + ] + }, + "ForestClassifier": { + "methods": [ + { + "name": "_set_oob_score", + "updates": [0] + }, + { + "name": "_validate_y_class_weight", + "updates": [0] + }, + "predict", + "predict_proba", + "predict_log_proba", + "apply", + "decision_path", + { + "name": "fit", + "updates": [0] + } + ] + }, + "ForestRegressor": { + "methods": [ + "predict", + { + "name": "_set_oob_score", + "updates": [0] + }, + "apply", + "decision_path", + { + "name": "fit", + "updates": [0] + } + ] + }, + "RandomForestClassifier": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "RandomForestClassifier" + }, + "predict", + "predict_proba", + "predict_log_proba", + "apply", + "decision_path", + { + "name": "fit", + "updates": [0] + } + ] + }, + "RandomForestRegressor": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "RandomForestRegressor" + }, + "predict", + "apply", + "decision_path", + { + "name": "fit", + "updates": [0] + } + ] + }, + "ExtraTreesClassifier": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "ExtraTreesClassifier" + }, + "predict", + "predict_proba", + "predict_log_proba", + "apply", + "decision_path", + { + "name": "fit", + "updates": [0] + } + ] + }, + "ExtraTreesRegressor": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "ExtraTreesRegressor" + }, + "predict", + "apply", + "decision_path", + { + "name": "fit", + "updates": [0] + } + ] + }, + "RandomTreesEmbedding": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "RandomTreesEmbedding" + }, + "_set_oob_score", + { + "name": "fit", + "updates": [0] + }, + { + "name": "fit_transform", + "updates": [0] + }, + "transform", + "apply", + "decision_path" + ] + } + } + }, + "gradient_boosting": { + "functions": [], + "types": { + "QuantileEstimator": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "QuantileEstimator" + }, + { + "name": "fit", + "updates": [0] + }, + "predict" + ] + }, + "MeanEstimator": { + "methods": [ + { + "name": "fit", + "updates": [0] + }, + "predict" + ] + }, + "LogOddsEstimator": { + "methods": [ + { + "name": "fit", + "updates": [0] + }, + "predict" + ] + }, + "ScaledLogOddsEstimator": { + "methods": [ + { + "name": "fit", + "updates": [0] + }, + "predict" + ] + }, + "PriorProbabilityEstimator": { + "methods": [ + { + "name": "fit", + "updates": [0] + }, + "predict" + ] + }, + "ZeroEstimator": { + "methods": [ + { + "name": "fit", + "updates": [0] + }, + "predict", + "predict_proba" + ] + }, + "LossFunction": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "LossFunction" + }, + "init_estimator", + { + "name": "update_terminal_regions", + "updates": [5] + } + ] + }, + "RegressionLossFunction": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "RegressionLossFunction" + }, + "__init__", + "init_estimator", + { + "name": "update_terminal_regions", + "updates": [5] + } + ] + }, + "LeastSquaresError": { + "methods": [ + "init_estimator", + "__call__", + "negative_gradient", + { + "name": "update_terminal_regions", + "updates": [5] + }, + "_update_terminal_region" + ] + }, + "LeastAbsoluteError": { + "methods": [ + "init_estimator", + "__call__", + "negative_gradient", + "_update_terminal_region", + { + "name": "update_terminal_regions", + "updates": [5] + } + ] + }, + "HuberLossFunction": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "HuberLossFunction" + }, + "init_estimator", + "__call__", + { + "name": "negative_gradient", + "updates": [0] + }, + "_update_terminal_region", + { + "name": "update_terminal_regions", + "updates": [5] + } + ] + }, + "QuantileLossFunction": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "QuantileLossFunction" + }, + "init_estimator", + "__call__", + "negative_gradient", + "_update_terminal_region", + { + "name": "update_terminal_regions", + "updates": [5] + } + ] + }, + "ClassificationLossFunction": { + "methods": [ + "_score_to_proba", + "init_estimator", + { + "name": "update_terminal_regions", + "updates": [5] + } + ] + }, + "BinomialDeviance": { + "methods": [ + { + "name": "__init__", + "updates": [], + "returns": "BinomialDeviance" + }, + "__init__", + "init_estimator", + "__call__", + "negative_gradient", + "_update_terminal_region", + "_score_to_proba", + "_score_to_decision", + { + "name": "update_terminal_regions", + "updates": [5] + } + ] + }, + "MultinomialDeviance": { + "methods": [ + { + "name": "__init__", + "updates": [], + "returns": "MultinomialDeviance" + }, + "__init__", + "init_estimator", + "__call__", + "negative_gradient", + "_update_terminal_region", + "_score_to_proba", + "_score_to_decision", + { + "name": "update_terminal_regions", + "updates": [5] + } + ] + }, + "ExponentialLoss": { + "methods": [ + { + "name": "__init__", + "updates": [], + "returns": "ExponentialLoss" + }, + "__init__", + "init_estimator", + "__call__", + "negative_gradient", + "_update_terminal_region", + "_score_to_proba", + "_score_to_decision", + { + "name": "update_terminal_regions", + "updates": [5] + } + ] + }, + "VerboseReporter": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "VerboseReporter" + }, + { + "name": "init", + "updates": [0] + }, + { + "name": "update", + "updates": [0] + } + ] + }, + "BaseGradientBoosting": { + "methods": [ + "_fit_stage", + { + "name": "_check_params", + "updates": [0] + }, + { + "name": "_init_state", + "updates": [0] + }, + { + "name": "_clear_state", + "updates": [0] + }, + { + "name": "_resize_state", + "updates": [0] + }, + "_is_initialized", + "_check_initialized", + { + "name": "fit", + "updates": [0] + }, + "_fit_stages", + "_make_estimator", + "_raw_predict_init", + "_raw_predict", + "_staged_raw_predict", + { + "name": "_validate_y", + "updates": [0] + }, + "apply" + ] + }, + "GradientBoostingClassifier": { + "methods": [ + { + "name": "__init__", + "updates": [], + "returns": "GradientBoostingClassifier" + }, + "__init__", + { + "name": "_validate_y", + "updates": [0] + }, + "decision_function", + "staged_decision_function", + "predict", + "staged_predict", + "predict_proba", + "predict_log_proba", + "staged_predict_proba", + { + "name": "fit", + "updates": [0] + }, + "apply" + ] + }, + "GradientBoostingRegressor": { + "methods": [ + { + "name": "__init__", + "updates": [], + "returns": "GradientBoostingRegressor" + }, + "__init__", + "predict", + "staged_predict", + "apply", + { + "name": "fit", + "updates": [0] + } + ] + } + } + }, + "iforest": { + "functions": ["_average_path_length"], + "types": { + "IsolationForest": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "IsolationForest" + }, + "_set_oob_score", + "_parallel_args", + { + "name": "fit", + "updates": [0] + }, + "predict", + "decision_function", + "score_samples", + "_compute_chunked_score_samples", + "_compute_score_samples" + ] + } + } + }, + "partial_dependence": { + "functions": ["_grid_from_X"], + "types": {} + }, + "setup": { + "functions": ["configuration"], + "types": {} + }, + "voting": { + "functions": ["_parallel_fit_estimator"], + "types": { + "_BaseVoting": { + "methods": ["_predict", "set_params", "get_params"] + }, + "VotingClassifier": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "VotingClassifier" + }, + { + "name": "fit", + "updates": [0] + }, + "predict", + "_collect_probas", + "_predict_proba", + "transform", + "set_params", + "get_params" + ] + }, + "VotingRegressor": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "VotingRegressor" + }, + "fit", + "predict", + "transform", + "set_params", + "get_params" + ] + } + } + }, + "weight_boosting": { + "functions": ["_samme_proba"], + "types": { + "BaseWeightBoosting": { + "methods": [ + "_validate_data", + { + "name": "fit", + "updates": [0, 3] + }, + "staged_score" + ] + }, + "AdaBoostClassifier": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "AdaBoostClassifier" + }, + { + "name": "fit", + "updates": [0, 3] + }, + "_validate_estimator", + { + "name": "_boost", + "updates": [0] + }, + { + "name": "_boost_real", + "updates": [0] + }, + { + "name": "_boost_discrete", + "updates": [0] + }, + "predict", + "staged_predict", + "decision_function", + "staged_decision_function", + "predict_proba", + "staged_predict_proba", + "predict_log_proba", + "staged_score" + ] + }, + "AdaBoostRegressor": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "AdaBoostRegressor" + }, + { + "name": "fit", + "updates": [0, 3] + }, + "_validate_estimator", + "_boost", + "_get_median_predict", + "predict", + "staged_predict", + "staged_score" + ] + } + } + } + }, + "types": { + "BaseEnsemble": { + "methods": [ + { + "name": "_validate_estimator", + "updates": [0] + }, + "_make_estimator", + "__len__", + "__getitem__", + "__iter__" + ] + }, + "RandomForestClassifier": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "RandomForestClassifier" + }, + "predict", + "predict_proba", + "predict_log_proba", + "apply", + "decision_path", + { + "name": "fit", + "updates": [0] + } + ] + }, + "RandomForestRegressor": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "RandomForestRegressor" + }, + "predict", + "apply", + "decision_path", + { + "name": "fit", + "updates": [0] + } + ] + }, + "RandomTreesEmbedding": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "RandomTreesEmbedding" + }, + "_set_oob_score", + { + "name": "fit", + "updates": [0] + }, + { + "name": "fit_transform", + "updates": [0] + }, + "transform", + "apply", + "decision_path" + ] + }, + "ExtraTreesClassifier": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "ExtraTreesClassifier" + }, + "predict", + "predict_proba", + "predict_log_proba", + "apply", + "decision_path", + { + "name": "fit", + "updates": [0] + } + ] + }, + "ExtraTreesRegressor": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "ExtraTreesRegressor" + }, + "predict", + "apply", + "decision_path", + { + "name": "fit", + "updates": [0] + } + ] + }, + "BaggingClassifier": { + "methods": [ + { + "name": "__init__", + "updates": [], + "returns": "BaggingClassifier" + }, + "__init__", + "_validate_estimator", + { + "name": "_set_oob_score", + "updates": [0] + }, + { + "name": "_validate_y", + "updates": [0] + }, + "predict", + "predict_proba", + "predict_log_proba", + { + "name": "fit", + "updates": [0] + } + ] + }, + "BaggingRegressor": { + "methods": [ + { + "name": "__init__", + "updates": [], + "returns": "BaggingRegressor" + }, + "__init__", + "predict", + "_validate_estimator", + { + "name": "_set_oob_score", + "updates": [0] + }, + { + "name": "fit", + "updates": [0] + } + ] + }, + "IsolationForest": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "IsolationForest" + }, + "_set_oob_score", + "_parallel_args", + { + "name": "fit", + "updates": [0] + }, + "predict", + "decision_function", + "score_samples", + "_compute_chunked_score_samples", + "_compute_score_samples" + ] + }, + "AdaBoostClassifier": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "AdaBoostClassifier" + }, + { + "name": "fit", + "updates": [0, 3] + }, + "_validate_estimator", + { + "name": "_boost", + "updates": [0] + }, + { + "name": "_boost_real", + "updates": [0] + }, + { + "name": "_boost_discrete", + "updates": [0] + }, + "predict", + "staged_predict", + "decision_function", + "staged_decision_function", + "predict_proba", + "staged_predict_proba", + "predict_log_proba", + "staged_score" + ] + }, + "AdaBoostRegressor": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "AdaBoostRegressor" + }, + { + "name": "fit", + "updates": [0, 3] + }, + "_validate_estimator", + "_boost", + "_get_median_predict", + "predict", + "staged_predict", + "staged_score" + ] + }, + "GradientBoostingClassifier": { + "methods": [ + { + "name": "__init__", + "updates": [], + "returns": "GradientBoostingClassifier" + }, + "__init__", + { + "name": "_validate_y", + "updates": [0] + }, + "decision_function", + "staged_decision_function", + "predict", + "staged_predict", + "predict_proba", + "predict_log_proba", + "staged_predict_proba", + { + "name": "fit", + "updates": [0] + }, + "apply" + ] + }, + "GradientBoostingRegressor": { + "methods": [ + { + "name": "__init__", + "updates": [], + "returns": "GradientBoostingRegressor" + }, + "__init__", + "predict", + "staged_predict", + "apply", + { + "name": "fit", + "updates": [0] + } + ] + }, + "VotingClassifier": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "VotingClassifier" + }, + { + "name": "fit", + "updates": [0] + }, + "predict", + "_collect_probas", + "_predict_proba", + "transform", + "set_params", + "get_params" + ] + }, + "VotingRegressor": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "VotingRegressor" + }, + "fit", + "predict", + "transform", + "set_params", + "get_params" + ] + } + }, + "functions": [] + }, + "experimental": { + "modules": { + "tests": { + "modules": { + "test_enable_hist_gradient_boosting": { + "functions": ["test_imports_strategies"], + "types": {} + }, + "test_enable_iterative_imputer": { + "functions": ["test_imports_strategies"], + "types": {} + } + }, + "types": {}, + "functions": [] + }, + "enable_hist_gradient_boosting": { + "functions": [], + "types": {} + }, + "enable_iterative_imputer": { + "functions": [], + "types": {} + } + }, + "types": {}, + "functions": [] + }, + "externals": { + "modules": { + "joblib": { + "modules": { + "numpy_pickle": { + "functions": [], + "types": {} + } + }, + "types": {}, + "functions": [] + }, + "conftest": { + "functions": ["pytest_ignore_collect"], + "types": {} + }, + "setup": { + "functions": ["configuration"], + "types": {} + }, + "six": { + "functions": [ + { + "name": "_add_doc", + "updates": [0] + }, + "_import_module", + "add_move", + "remove_move", + "iterkeys", + "itervalues", + "iteritems", + "iterlists", + "with_metaclass", + "add_metaclass" + ], + "types": { + "X": { + "methods": ["__len__"] + }, + "_LazyDescr": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "_LazyDescr" + }, + "__get__" + ] + }, + "MovedModule": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "MovedModule" + }, + "_resolve" + ] + }, + "MovedAttribute": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "MovedAttribute" + }, + "_resolve" + ] + }, + "_MovedItems": { + "methods": [] + }, + "Module_six_moves_urllib_parse": { + "methods": [] + }, + "Module_six_moves_urllib_error": { + "methods": [] + }, + "Module_six_moves_urllib_request": { + "methods": [] + }, + "Module_six_moves_urllib_response": { + "methods": [] + }, + "Module_six_moves_urllib_robotparser": { + "methods": [] + }, + "Module_six_moves_urllib": { + "methods": [] + }, + "Iterator": { + "methods": ["next"] + } + } + } + }, + "types": {}, + "functions": [] + }, + "feature_extraction": { + "modules": { + "tests": { + "modules": { + "test_dict_vectorizer": { + "functions": [ + "test_feature_selection", + "test_one_of_k", + "test_unseen_or_no_features", + "test_deterministic_vocabulary" + ], + "types": {} + }, + "test_feature_hasher": { + "functions": [ + "test_feature_hasher_dicts", + "test_feature_hasher_strings", + "test_feature_hasher_pairs", + "test_feature_hasher_pairs_with_string_values", + "test_hash_empty_input", + "test_hasher_invalid_input", + "test_hasher_set_params", + "test_hasher_zeros", + "test_hash_collisions" + ], + "types": {} + }, + "test_image": { + "functions": [ + "test_img_to_graph", + "test_grid_to_graph", + "_downsampled_face", + "_orange_face", + "_make_images", + "test_extract_patches_all", + "test_extract_patches_all_color", + "test_extract_patches_all_rect", + "test_extract_patches_max_patches", + "test_extract_patch_same_size_image", + "test_extract_patches_less_than_max_patches", + "test_reconstruct_patches_perfect", + "test_reconstruct_patches_perfect_color", + "test_patch_extractor_fit", + "test_patch_extractor_max_patches", + "test_patch_extractor_max_patches_default", + "test_patch_extractor_all_patches", + "test_patch_extractor_color", + "test_extract_patches_strided", + "test_extract_patches_square", + "test_width_patch" + ], + "types": {} + }, + "test_text": { + "functions": [ + "uppercase", + "strip_eacute", + "split_tokenize", + "lazy_analyze", + "test_strip_accents", + "test_to_ascii", + "test_word_analyzer_unigrams_and_bigrams", + "test_unicode_decode_error", + "test_char_ngram_analyzer", + "test_char_wb_ngram_analyzer", + "test_word_ngram_analyzer", + "test_countvectorizer_custom_vocabulary", + "test_countvectorizer_custom_vocabulary_pipeline", + "test_countvectorizer_custom_vocabulary_repeated_indices", + "test_countvectorizer_custom_vocabulary_gap_index", + "test_countvectorizer_stop_words", + "test_countvectorizer_empty_vocabulary", + "test_fit_countvectorizer_twice", + "test_tf_idf_smoothing", + "test_tfidf_no_smoothing", + "test_sublinear_tf", + "test_vectorizer", + "test_tfidf_vectorizer_setters", + "test_feature_names", + "test_count_vectorizer_max_features", + "test_vectorizer_max_df", + "test_vectorizer_min_df", + "test_count_binary_occurrences", + "test_vectorizer_pipeline_cross_validation", + "test_tfidf_vectorizer_with_fixed_vocabulary", + "test_pickling_vectorizer", + "test_countvectorizer_vocab_sets_when_pickling", + "test_countvectorizer_vocab_dicts_when_pickling", + "test_stop_words_removal", + "test_pickling_transformer", + "test_transformer_idf_setter", + "test_tfidf_vectorizer_setter", + "test_tfidfvectorizer_invalid_idf_attr", + "test_non_unique_vocab", + "test_tfidfvectorizer_binary", + "test_tfidfvectorizer_export_idf", + "test_vectorizer_vocab_clone", + "test_tfidf_transformer_sparse", + "_check_stop_words_consistency" + ], + "types": { + "CustomEstimator": { + "methods": ["build_preprocessor"] + } + } + } + }, + "types": {}, + "functions": [] + }, + "dict_vectorizer": { + "functions": ["_tosequence"], + "types": { + "DictVectorizer": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "DictVectorizer" + }, + { + "name": "fit", + "updates": [0] + }, + { + "name": "_transform", + "updates": [0] + }, + { + "name": "fit_transform", + "updates": [0] + }, + "inverse_transform", + { + "name": "transform", + "updates": [0] + }, + "get_feature_names", + { + "name": "restrict", + "updates": [0] + }, + "_more_tags" + ] + } + } + }, + "hashing": { + "functions": ["_iteritems"], + "types": { + "FeatureHasher": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "FeatureHasher" + }, + "fit", + "transform", + "_more_tags" + ] + } + } + }, + "image": { + "functions": [ + "_make_edges_3d", + "_compute_gradient_3d", + "_mask_edges_weights", + "_to_graph", + "img_to_graph", + "grid_to_graph", + "_compute_n_patches", + "extract_patches", + "extract_patches_2d", + "reconstruct_from_patches_2d" + ], + "types": { + "PatchExtractor": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "PatchExtractor" + }, + "fit", + { + "name": "transform", + "updates": [0] + }, + "_more_tags" + ] + } + } + }, + "setup": { + "functions": ["configuration"], + "types": {} + }, + "stop_words": { + "functions": [], + "types": {} + }, + "text": { + "functions": [ + "strip_accents_unicode", + "strip_accents_ascii", + "strip_tags", + "_check_stop_list", + "_document_frequency", + "_make_int_array" + ], + "types": { + "VectorizerMixin": { + "methods": [ + "decode", + "_word_ngrams", + "_char_ngrams", + "_char_wb_ngrams", + "build_preprocessor", + "build_tokenizer", + "get_stop_words", + { + "name": "_check_stop_words_consistency", + "updates": [0] + }, + "_validate_custom_analyzer", + { + "name": "build_analyzer", + "updates": [0] + }, + { + "name": "_validate_vocabulary", + "updates": [0] + }, + "_check_vocabulary", + "_validate_params" + ] + }, + "HashingVectorizer": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "HashingVectorizer" + }, + "partial_fit", + "fit", + "transform", + "fit_transform", + "_get_hasher", + "_more_tags", + "decode", + "build_preprocessor", + "build_tokenizer", + "get_stop_words", + { + "name": "build_analyzer", + "updates": [0] + } + ] + }, + "CountVectorizer": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "CountVectorizer" + }, + { + "name": "_sort_features", + "updates": [1, 2] + }, + { + "name": "_limit_features", + "updates": [2] + }, + "_count_vocab", + { + "name": "fit", + "updates": [0] + }, + { + "name": "fit_transform", + "updates": [0] + }, + "transform", + "inverse_transform", + "get_feature_names", + "_more_tags", + "decode", + "build_preprocessor", + "build_tokenizer", + "get_stop_words", + { + "name": "build_analyzer", + "updates": [0] + } + ] + }, + "TfidfTransformer": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "TfidfTransformer" + }, + { + "name": "fit", + "updates": [0] + }, + { + "name": "transform", + "updates": [1] + }, + "_more_tags" + ] + }, + "TfidfVectorizer": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "TfidfVectorizer" + }, + "_check_params", + { + "name": "fit", + "updates": [0] + }, + { + "name": "fit_transform", + "updates": [0] + }, + "transform", + "_more_tags", + "inverse_transform", + "get_feature_names", + "decode", + "build_preprocessor", + "build_tokenizer", + "get_stop_words", + { + "name": "build_analyzer", + "updates": [0] + } + ] + } + } + } + }, + "types": { + "DictVectorizer": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "DictVectorizer" + }, + { + "name": "fit", + "updates": [0] + }, + { + "name": "_transform", + "updates": [0] + }, + { + "name": "fit_transform", + "updates": [0] + }, + "inverse_transform", + { + "name": "transform", + "updates": [0] + }, + "get_feature_names", + { + "name": "restrict", + "updates": [0] + }, + "_more_tags" + ] + }, + "FeatureHasher": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "FeatureHasher" + }, + "fit", + "transform", + "_more_tags" + ] + } + }, + "functions": ["img_to_graph", "grid_to_graph"] + }, + "feature_selection": { + "modules": { + "tests": { + "modules": { + "test_base": { + "functions": [ + "test_transform_dense", + "test_transform_sparse", + "test_inverse_transform_dense", + "test_inverse_transform_sparse", + "test_get_support" + ], + "types": { + "StepSelector": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "StepSelector" + }, + { + "name": "fit", + "updates": [0] + }, + "_get_support_mask" + ] + } + } + }, + "test_chi2": { + "functions": [ + "mkchi2", + "test_chi2", + "test_chi2_coo", + "test_chi2_negative", + "test_chi2_unused_feature", + "test_chisquare" + ], + "types": {} + }, + "test_feature_select": { + "functions": [ + "test_f_oneway_vs_scipy_stats", + "test_f_oneway_ints", + "test_f_classif", + "test_f_regression", + "test_f_regression_input_dtype", + "test_f_regression_center", + "test_f_classif_multi_class", + "test_select_percentile_classif", + "test_select_percentile_classif_sparse", + "test_select_kbest_classif", + "test_select_kbest_all", + "test_select_kbest_zero", + "test_select_heuristics_classif", + "assert_best_scores_kept", + "test_select_percentile_regression", + "test_select_percentile_regression_full", + "test_invalid_percentile", + "test_select_kbest_regression", + "test_select_heuristics_regression", + "test_boundary_case_ch2", + "test_select_fdr_regression", + "test_select_fwe_regression", + "test_selectkbest_tiebreaking", + "test_selectpercentile_tiebreaking", + "test_tied_pvalues", + "test_scorefunc_multilabel", + "test_tied_scores", + "test_nans", + "test_score_func_error", + "test_invalid_k", + "test_f_classif_constant_feature", + "test_no_feature_selected", + "test_mutual_info_classif", + "test_mutual_info_regression" + ], + "types": {} + }, + "test_from_model": { + "functions": [ + "test_max_features", + "test_max_features_tiebreak", + "test_threshold_and_max_features", + "test_coef_default_threshold", + "test_calling_fit_reinitializes", + "test_threshold_string" + ], + "types": { + "FixedImportanceEstimator": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "FixedImportanceEstimator" + }, + { + "name": "fit", + "updates": [0] + } + ] + } + } + }, + "test_mutual_info": { + "functions": [ + "test_compute_mi_dd", + "test_compute_mi_cc", + "test_compute_mi_cd", + "test_compute_mi_cd_unique_label", + "test_mutual_info_classif_discrete", + "test_mutual_info_regression", + "test_mutual_info_classif_mixed", + "test_mutual_info_options" + ], + "types": {} + }, + "test_rfe": { + "functions": [ + "test_rfe_features_importance", + "test_rfe", + "test_rfe_mockclassifier", + "test_rfecv", + "test_rfecv_mockclassifier", + "test_rfecv_verbose_output", + "test_rfecv_grid_scores_size", + "test_rfe_min_step", + "test_number_of_subsets_of_features" + ], + "types": { + "MockClassifier": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "MockClassifier" + }, + { + "name": "fit", + "updates": [0] + }, + "predict", + "score", + "get_params", + "set_params" + ] + } + } + }, + "test_variance_threshold": { + "functions": ["test_zero_variance", "test_variance_threshold"], + "types": {} + } + }, + "types": {}, + "functions": [] + }, + "base": { + "functions": [], + "types": { + "SelectorMixin": { + "methods": ["get_support", "transform", "inverse_transform"] + } + } + }, + "from_model": { + "functions": ["_get_feature_importances", "_calculate_threshold"], + "types": { + "SelectFromModel": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "SelectFromModel" + }, + "_get_support_mask", + { + "name": "fit", + "updates": [0] + } + ] + } + } + }, + "mutual_info_": { + "functions": [ + "_compute_mi_cc", + "_compute_mi_cd", + "_compute_mi", + "_iterate_columns", + { + "name": "_estimate_mi", + "updates": [0] + }, + "mutual_info_regression", + "mutual_info_classif" + ], + "types": {} + }, + "rfe": { + "functions": ["_rfe_single_fit"], + "types": { + "RFE": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "RFE" + }, + { + "name": "fit", + "updates": [0] + }, + { + "name": "_fit", + "updates": [0] + }, + "_get_support_mask", + "_more_tags" + ] + }, + "RFECV": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "RFECV" + }, + { + "name": "fit", + "updates": [0] + } + ] + } + } + }, + "univariate_selection": { + "functions": [ + { + "name": "_clean_nans", + "updates": [0] + }, + "f_oneway", + "f_classif", + "_chisquare", + "chi2", + "f_regression" + ], + "types": { + "_BaseFilter": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "_BaseFilter" + }, + { + "name": "fit", + "updates": [0] + }, + "_check_params" + ] + }, + "SelectPercentile": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "SelectPercentile" + }, + "_check_params", + "_get_support_mask", + { + "name": "fit", + "updates": [0] + } + ] + }, + "SelectKBest": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "SelectKBest" + }, + "_check_params", + "_get_support_mask", + { + "name": "fit", + "updates": [0] + } + ] + }, + "SelectFpr": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "SelectFpr" + }, + "_get_support_mask", + { + "name": "fit", + "updates": [0] + } + ] + }, + "SelectFdr": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "SelectFdr" + }, + "_get_support_mask", + { + "name": "fit", + "updates": [0] + } + ] + }, + "SelectFwe": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "SelectFwe" + }, + "_get_support_mask", + { + "name": "fit", + "updates": [0] + } + ] + }, + "GenericUnivariateSelect": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "GenericUnivariateSelect" + }, + "_make_selector", + "_check_params", + "_get_support_mask", + { + "name": "fit", + "updates": [0] + } + ] + } + } + }, + "variance_threshold": { + "functions": [], + "types": { + "VarianceThreshold": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "VarianceThreshold" + }, + { + "name": "fit", + "updates": [0] + }, + "_get_support_mask" + ] + } + } + } + }, + "types": { + "SelectPercentile": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "SelectPercentile" + }, + "_check_params", + "_get_support_mask", + { + "name": "fit", + "updates": [0] + } + ] + }, + "SelectKBest": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "SelectKBest" + }, + "_check_params", + "_get_support_mask", + { + "name": "fit", + "updates": [0] + } + ] + }, + "SelectFpr": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "SelectFpr" + }, + "_get_support_mask", + { + "name": "fit", + "updates": [0] + } + ] + }, + "SelectFdr": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "SelectFdr" + }, + "_get_support_mask", + { + "name": "fit", + "updates": [0] + } + ] + }, + "SelectFwe": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "SelectFwe" + }, + "_get_support_mask", + { + "name": "fit", + "updates": [0] + } + ] + }, + "GenericUnivariateSelect": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "GenericUnivariateSelect" + }, + "_make_selector", + "_check_params", + "_get_support_mask", + { + "name": "fit", + "updates": [0] + } + ] + }, + "VarianceThreshold": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "VarianceThreshold" + }, + { + "name": "fit", + "updates": [0] + }, + "_get_support_mask" + ] + }, + "RFE": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "RFE" + }, + { + "name": "fit", + "updates": [0] + }, + { + "name": "_fit", + "updates": [0] + }, + "_get_support_mask", + "_more_tags" + ] + }, + "RFECV": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "RFECV" + }, + { + "name": "fit", + "updates": [0] + } + ] + }, + "SelectFromModel": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "SelectFromModel" + }, + "_get_support_mask", + { + "name": "fit", + "updates": [0] + } + ] + } + }, + "functions": [ + "chi2", + "f_classif", + "f_oneway", + "f_regression", + "mutual_info_regression", + "mutual_info_classif" + ] + }, + "gaussian_process": { + "modules": { + "tests": { + "modules": { + "test_gpc": { + "functions": ["f", "test_random_starts"], + "types": {} + }, + "test_gpr": { + "functions": [ + "f", + "test_no_optimizer", + "test_anisotropic_kernel", + "test_random_starts", + "test_y_multioutput", + "test_gpr_correct_error_message", + "test_no_fit_default_predict" + ], + "types": {} + }, + "test_kernels": { + "functions": [ + "test_kernel_operator_commutative", + "test_kernel_anisotropic", + "check_hyperparameters_equal", + "test_matern_kernel" + ], + "types": {} + } + }, + "types": {}, + "functions": [] + }, + "correlation_models": { + "functions": [], + "types": {} + }, + "gpc": { + "functions": [], + "types": { + "_BinaryGaussianProcessClassifierLaplace": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "_BinaryGaussianProcessClassifierLaplace" + }, + { + "name": "fit", + "updates": [0] + }, + "predict", + "predict_proba", + { + "name": "log_marginal_likelihood", + "updates": [0] + }, + { + "name": "_posterior_mode", + "updates": [0] + }, + "_constrained_optimization" + ] + }, + "GaussianProcessClassifier": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "GaussianProcessClassifier" + }, + { + "name": "fit", + "updates": [0] + }, + "predict", + "predict_proba", + "log_marginal_likelihood" + ] + } + } + }, + "gpr": { + "functions": [], + "types": { + "GaussianProcessRegressor": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "GaussianProcessRegressor" + }, + { + "name": "fit", + "updates": [0] + }, + { + "name": "predict", + "updates": [0] + }, + { + "name": "sample_y", + "updates": [0] + }, + "log_marginal_likelihood", + "_constrained_optimization" + ] + } + } + }, + "kernels": { + "functions": ["_check_length_scale", "_approx_fprime"], + "types": { + "Hyperparameter": { + "methods": ["__new__", "__eq__"] + }, + "Kernel": { + "methods": [ + "get_params", + "set_params", + "clone_with_theta", + "__add__", + "__radd__", + "__mul__", + "__rmul__", + "__pow__", + "__eq__", + "__repr__" + ] + }, + "NormalizedKernelMixin": { + "methods": ["diag"] + }, + "StationaryKernelMixin": { + "methods": ["is_stationary"] + }, + "CompoundKernel": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "CompoundKernel" + }, + "get_params", + "__call__", + "__eq__", + "is_stationary", + "diag", + "set_params", + "clone_with_theta" + ] + }, + "KernelOperator": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "KernelOperator" + }, + "get_params", + "__eq__", + "is_stationary", + "set_params", + "clone_with_theta" + ] + }, + "Sum": { + "methods": [ + "__call__", + "diag", + "__repr__", + "get_params", + "is_stationary", + "set_params", + "clone_with_theta" + ] + }, + "Product": { + "methods": [ + "__call__", + "diag", + "__repr__", + "get_params", + "is_stationary", + "set_params", + "clone_with_theta" + ] + }, + "Exponentiation": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "Exponentiation" + }, + "get_params", + "__eq__", + "__call__", + "diag", + "__repr__", + "is_stationary", + "set_params", + "clone_with_theta" + ] + }, + "ConstantKernel": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "ConstantKernel" + }, + "__call__", + "diag", + "__repr__", + "is_stationary", + "get_params", + "set_params", + "clone_with_theta" + ] + }, + "WhiteKernel": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "WhiteKernel" + }, + "__call__", + "diag", + "__repr__", + "is_stationary", + "get_params", + "set_params", + "clone_with_theta" + ] + }, + "RBF": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "RBF" + }, + "__call__", + "__repr__", + "is_stationary", + "diag", + "get_params", + "set_params", + "clone_with_theta" + ] + }, + "Matern": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "Matern" + }, + "__call__", + "__repr__", + "is_stationary", + "diag", + "get_params", + "set_params", + "clone_with_theta" + ] + }, + "RationalQuadratic": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "RationalQuadratic" + }, + "__call__", + "__repr__", + "is_stationary", + "diag", + "get_params", + "set_params", + "clone_with_theta" + ] + }, + "ExpSineSquared": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "ExpSineSquared" + }, + "__call__", + "__repr__", + "is_stationary", + "diag", + "get_params", + "set_params", + "clone_with_theta" + ] + }, + "DotProduct": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "DotProduct" + }, + "__call__", + "diag", + "is_stationary", + "__repr__", + "get_params", + "set_params", + "clone_with_theta" + ] + }, + "PairwiseKernel": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "PairwiseKernel" + }, + "__call__", + "diag", + "is_stationary", + "__repr__", + "get_params", + "set_params", + "clone_with_theta" + ] + } + } + }, + "regression_models": { + "functions": [], + "types": {} + } + }, + "types": { + "GaussianProcessRegressor": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "GaussianProcessRegressor" + }, + { + "name": "fit", + "updates": [0] + }, + { + "name": "predict", + "updates": [0] + }, + { + "name": "sample_y", + "updates": [0] + }, + "log_marginal_likelihood", + "_constrained_optimization" + ] + }, + "GaussianProcessClassifier": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "GaussianProcessClassifier" + }, + { + "name": "fit", + "updates": [0] + }, + "predict", + "predict_proba", + "log_marginal_likelihood" + ] + } + }, + "functions": [] + }, + "impute": { + "modules": { + "tests": { + "modules": { + "test_impute": { + "functions": [ + "_check_statistics", + "test_imputation_shape", + "safe_median", + "safe_mean", + "test_imputation_mean_median", + "test_imputation_median_special_cases", + "test_imputation_most_frequent", + "test_imputation_constant_integer", + "test_imputation_copy", + "test_iterative_imputer_zero_iters", + "test_iterative_imputer_verbose", + "test_iterative_imputer_all_missing", + "test_iterative_imputer_clip", + "test_iterative_imputer_clip_truncnorm", + "test_iterative_imputer_truncated_normal_posterior", + "test_iterative_imputer_transform_stochasticity", + "test_iterative_imputer_no_missing", + "test_iterative_imputer_rank_one", + "test_iterative_imputer_additive_matrix", + "test_iterative_imputer_early_stopping", + "test_iterative_imputer_catch_warning", + "test_missing_indicator_string", + "test_missing_indicator_no_missing", + "test_missing_indicator_sparse_no_explicit_zeros" + ], + "types": {} + } + }, + "types": {}, + "functions": [] + } + }, + "types": {}, + "functions": [] + }, + "inspection": { + "modules": { + "tests": { + "modules": { + "test_partial_dependence": { + "functions": [ + "test_grid_from_X", + "test_warning_recursion_non_constant_init", + "test_partial_dependence_sample_weight", + "test_plot_partial_dependence", + "test_plot_partial_dependence_multiclass", + "test_plot_partial_dependence_multioutput", + "test_plot_partial_dependence_fig" + ], + "types": { + "NoPredictProbaNoDecisionFunction": { + "methods": [ + { + "name": "fit", + "updates": [0] + } + ] + } + } + } + }, + "types": {}, + "functions": [] + }, + "partial_dependence": { + "functions": [ + "_grid_from_X", + "_partial_dependence_recursion", + "_partial_dependence_brute", + "partial_dependence", + "plot_partial_dependence" + ], + "types": {} + } + }, + "types": {}, + "functions": ["partial_dependence", "plot_partial_dependence"] + }, + "linear_model": { + "modules": { + "tests": { + "modules": { + "test_base": { + "functions": [ + "test_linear_regression", + "test_linear_regression_sample_weights", + "test_raises_value_error_if_sample_weights_greater_than_1d", + "test_fit_intercept", + "test_linear_regression_sparse", + "test_linear_regression_multiple_outcome", + "test_linear_regression_sparse_multiple_outcome", + "test_preprocess_data", + "test_preprocess_data_multioutput", + "test_preprocess_data_weighted", + "test_sparse_preprocess_data_with_return_mean", + "test_csr_preprocess_data", + "test_dtype_preprocess_data", + "test_rescale_data", + "test_fused_types_make_dataset" + ], + "types": {} + }, + "test_bayes": { + "functions": [ + "test_n_iter", + "test_bayesian_ridge_scores", + "test_bayesian_ridge_score_values", + "test_bayesian_ridge_parameter", + "test_bayesian_sample_weights", + "test_toy_bayesian_ridge_object", + "test_prediction_bayesian_ridge_ard_with_constant_input", + "test_std_bayesian_ridge_ard_with_constant_input", + "test_update_of_sigma_in_ard", + "test_toy_ard_object", + "test_ard_accuracy_on_easy_problem", + "test_return_std" + ], + "types": {} + }, + "test_coordinate_descent": { + "functions": [ + "test_lasso_zero", + "test_lasso_toy", + "test_enet_toy", + "build_dataset", + "test_lasso_cv_with_some_model_selection", + "test_lasso_cv_positive_constraint", + "test_lasso_path_return_models_vs_new_return_gives_same_coefficients", + "test_warm_start", + "test_lasso_alpha_warning", + "test_lasso_positive_constraint", + "test_enet_positive_constraint", + "test_enet_cv_positive_constraint", + "test_multi_task_lasso_and_enet", + "test_lasso_readonly_data", + "test_multi_task_lasso_readonly_data", + "test_enet_multitarget", + "test_multioutput_enetcv_error", + "test_warm_start_convergence", + "test_warm_start_convergence_with_regularizer_decrement", + "test_random_descent", + "test_enet_path_positive", + "test_sparse_dense_descent_paths", + "test_check_input_false", + "test_enet_copy_X_False_check_input_False", + "test_overrided_gram_matrix", + "test_enet_float_precision", + "test_enet_l1_ratio", + "test_coef_shape_not_zero", + "test_warm_start_multitask_lasso", + "test_convergence_warnings", + "test_sparse_input_convergence_warning" + ], + "types": {} + }, + "test_huber": { + "functions": [ + "make_regression_with_outliers", + "test_huber_equals_lr_for_high_epsilon", + "test_huber_max_iter", + "test_huber_gradient", + "test_huber_sample_weights", + "test_huber_sparse", + "test_huber_scaling_invariant", + "test_huber_warm_start", + "test_huber_better_r2_score", + "test_huber_bool" + ], + "types": {} + }, + "test_least_angle": { + "functions": [ + "test_simple", + "test_simple_precomputed", + "_assert_same_lars_path_result", + "test_x_none_gram_none_raises_value_error", + "test_all_precomputed", + "test_collinearity", + "test_no_path", + "test_no_path_precomputed", + "test_no_path_all_precomputed", + "test_singular_matrix", + "test_rank_deficient_design", + "test_lasso_lars_vs_lasso_cd", + "test_lasso_lars_vs_lasso_cd_early_stopping", + "test_lasso_lars_path_length", + "test_lasso_lars_vs_lasso_cd_ill_conditioned", + "test_lasso_lars_vs_lasso_cd_ill_conditioned2", + "test_lars_add_features", + "test_lars_n_nonzero_coefs", + "test_lars_cv_max_iter", + "test_lasso_lars_ic", + "test_lars_path_readonly_data", + "test_lasso_lars_vs_lasso_cd_positive", + "test_lasso_lars_vs_R_implementation" + ], + "types": {} + }, + "test_logistic": { + "functions": [ + "check_predictions", + "test_logistic_cv_score_does_not_warn_by_default", + "test_predict_iris", + "test_multinomial_binary_probabilities", + "test_consistency_path", + "test_logistic_regression_path_convergence_fail", + "test_liblinear_dual_random_state", + "test_logistic_loss_and_grad", + "test_logistic_grad_hess", + "test_intercept_logistic_helper", + "test_ovr_multinomial_iris", + "test_logistic_regression_solvers", + "test_logistic_regression_solvers_multiclass", + "_compute_class_weight_dictionary", + "test_logistic_regression_class_weights", + "test_multinomial_grad_hess", + "test_liblinear_decision_function_zero", + "test_logreg_intercept_scaling", + "test_logreg_l1", + "test_logreg_l1_sparse_data", + "test_logreg_predict_proba_multinomial", + "test_max_iter", + "test_saga_vs_liblinear", + "test_warm_start_converge_LR", + "test_elastic_net_coeffs", + "test_LogisticRegressionCV_GridSearchCV_elastic_net_ovr", + "test_logistic_regression_path_coefs_multinomial", + "test_logistic_regression_path_deprecation" + ], + "types": { + "MockScorer": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "MockScorer" + }, + { + "name": "__call__", + "updates": [0] + } + ] + } + } + }, + "test_omp": { + "functions": [ + "test_correct_shapes", + "test_correct_shapes_gram", + "test_n_nonzero_coefs", + "test_tol", + "test_with_without_gram", + "test_with_without_gram_tol", + "test_unreachable_accuracy", + "test_bad_input", + "test_perfect_signal_recovery", + "test_orthogonal_mp_gram_readonly", + "test_estimator", + "test_identical_regressors", + "test_swapped_regressors", + "test_no_atoms", + "test_omp_path", + "test_omp_return_path_prop_with_gram", + "test_omp_cv", + "test_omp_reaches_least_squares" + ], + "types": {} + }, + "test_passive_aggressive": { + "functions": [ + "test_classifier_undefined_methods", + "test_regressor_undefined_methods" + ], + "types": { + "MyPassiveAggressive": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "MyPassiveAggressive" + }, + { + "name": "fit", + "updates": [0] + }, + "project" + ] + } + } + }, + "test_perceptron": { + "functions": ["test_undefined_methods"], + "types": { + "MyPerceptron": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "MyPerceptron" + }, + { + "name": "fit", + "updates": [0] + }, + "project", + "predict" + ] + } + } + }, + "test_ransac": { + "functions": [ + "test_ransac_inliers_outliers", + "test_ransac_is_data_valid", + "test_ransac_is_model_valid", + "test_ransac_max_trials", + "test_ransac_stop_n_inliers", + "test_ransac_stop_score", + "test_ransac_score", + "test_ransac_predict", + "test_ransac_resid_thresh_no_inliers", + "test_ransac_no_valid_data", + "test_ransac_no_valid_model", + "test_ransac_exceed_max_skips", + "test_ransac_warn_exceed_max_skips", + "test_ransac_sparse_coo", + "test_ransac_sparse_csr", + "test_ransac_sparse_csc", + "test_ransac_none_estimator", + "test_ransac_min_n_samples", + "test_ransac_default_residual_threshold", + "test_ransac_dynamic_max_trials", + "test_ransac_fit_sample_weight" + ], + "types": {} + }, + "test_ridge": { + "functions": [ + "test_primal_dual_relationship", + "test_ridge_singular", + "test_ridge_regression_sample_weights", + "test_ridge_regression_convergence_fail", + "test_ridge_sample_weights", + "test_ridge_shapes", + "test_ridge_intercept", + "test_toy_ridge_object", + "test_ridge_vs_lstsq", + "test_ridge_individual_penalties", + "_make_sparse_offset_regression", + "_test_ridge_loo", + "_test_ridge_cv_normalize", + "_test_ridge_cv", + "_test_ridge_diabetes", + "_test_multi_ridge_diabetes", + "_test_ridge_classifiers", + "_test_tolerance", + "check_dense_sparse", + "test_ridge_sparse_svd", + "test_class_weights", + "test_raises_value_error_if_sample_weights_greater_than_1d", + "test_sparse_design_with_sample_weights", + "test_raises_value_error_if_solver_not_supported", + "test_sparse_cg_max_iter", + "test_ridge_fit_intercept_sparse", + "test_ridge_classifier_no_support_multilabel", + "test_dtype_match_cholesky" + ], + "types": {} + }, + "test_sag": { + "functions": [ + "log_dloss", + "log_loss", + "squared_dloss", + "squared_loss", + "get_pobj", + "sag", + "sag_sparse", + "get_step_size", + "test_classifier_matching", + "test_regressor_matching", + "test_get_auto_step_size", + "test_sag_regressor", + "test_multinomial_loss", + "test_multinomial_loss_ground_truth" + ], + "types": {} + }, + "test_sgd": { + "functions": [ + { + "name": "_update_kwargs", + "updates": [0] + }, + "SGDClassifier", + "SGDRegressor", + "SparseSGDClassifier", + "SparseSGDRegressor", + "asgd", + "_test_warm_start", + "test_l1_ratio", + "test_underflow_or_overlow", + "test_numerical_stability_large_gradient", + "test_tol_parameter", + "_test_gradient_common", + "test_gradient_hinge", + "test_gradient_squared_hinge", + "test_gradient_log", + "test_gradient_squared_loss", + "test_gradient_huber", + "test_gradient_modified_huber", + "test_gradient_epsilon_insensitive", + "test_gradient_squared_epsilon_insensitive", + "test_multi_thread_multi_class_and_early_stopping", + "test_multi_core_gridsearch_and_early_stopping" + ], + "types": { + "_SparseSGDClassifier": { + "methods": [ + "fit", + "partial_fit", + "decision_function", + "predict_proba" + ] + }, + "_SparseSGDRegressor": { + "methods": ["fit", "partial_fit", "decision_function"] + } + } + }, + "test_sparse_coordinate_descent": { + "functions": [ + "test_sparse_coef", + "test_normalize_option", + "test_lasso_zero", + "test_enet_toy_list_input", + "test_enet_toy_explicit_sparse_input", + "make_sparse_data", + "_test_sparse_enet_not_as_toy_dataset", + "test_sparse_enet_not_as_toy_dataset", + "test_sparse_lasso_not_as_toy_dataset", + "test_enet_multitarget", + "test_path_parameters", + "test_same_output_sparse_dense_lasso_and_enet_cv", + "test_same_multiple_output_sparse_dense", + "test_sparse_enet_coordinate_descent" + ], + "types": {} + }, + "test_theil_sen": { + "functions": [ + "gen_toy_problem_1d", + "gen_toy_problem_2d", + "gen_toy_problem_4d", + "test_modweiszfeld_step_1d", + "test_modweiszfeld_step_2d", + "test_spatial_median_1d", + "test_spatial_median_2d", + "test_theil_sen_1d", + "test_theil_sen_1d_no_intercept", + "test_theil_sen_2d", + "test_calc_breakdown_point", + "test_checksubparams_negative_subpopulation", + "test_checksubparams_too_few_subsamples", + "test_checksubparams_too_many_subsamples", + "test_checksubparams_n_subsamples_if_less_samples_than_features", + "test_subpopulation", + "test_subsamples", + "test_verbosity", + "test_theil_sen_parallel", + "test_less_samples_than_features" + ], + "types": {} + } + }, + "types": {}, + "functions": [] + }, + "base": { + "functions": [ + "make_dataset", + "_preprocess_data", + "_rescale_data", + "_pre_fit" + ], + "types": { + "LinearModel": { + "methods": [ + "_decision_function", + "predict", + { + "name": "_set_intercept", + "updates": [0] + } + ] + }, + "LinearClassifierMixin": { + "methods": ["decision_function", "predict", "_predict_proba_lr"] + }, + "SparseCoefMixin": { + "methods": [ + { + "name": "densify", + "updates": [0] + }, + { + "name": "sparsify", + "updates": [0] + } + ] + }, + "LinearRegression": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "LinearRegression" + }, + { + "name": "fit", + "updates": [0] + }, + "predict" + ] + } + } + }, + "bayes": { + "functions": [], + "types": { + "BayesianRidge": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "BayesianRidge" + }, + { + "name": "fit", + "updates": [0] + }, + "predict", + "_update_coef_", + "_log_marginal_likelihood" + ] + }, + "ARDRegression": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "ARDRegression" + }, + { + "name": "fit", + "updates": [0] + }, + "predict" + ] + } + } + }, + "coordinate_descent": { + "functions": [ + "_alpha_grid", + "lasso_path", + "enet_path", + { + "name": "_path_residuals", + "updates": [5] + } + ], + "types": { + "ElasticNet": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "ElasticNet" + }, + { + "name": "fit", + "updates": [0] + }, + "_decision_function" + ] + }, + "Lasso": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "Lasso" + }, + "__init__", + { + "name": "fit", + "updates": [0] + } + ] + }, + "LinearModelCV": { + "methods": [ + { + "name": "fit", + "updates": [0] + } + ] + }, + "LassoCV": { + "methods": [ + { + "name": "__init__", + "updates": [], + "returns": "LassoCV" + }, + "__init__", + { + "name": "fit", + "updates": [0] + } + ] + }, + "ElasticNetCV": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "ElasticNetCV" + }, + { + "name": "fit", + "updates": [0] + } + ] + }, + "MultiTaskElasticNet": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "MultiTaskElasticNet" + }, + { + "name": "fit", + "updates": [0] + }, + "_more_tags" + ] + }, + "MultiTaskLasso": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "MultiTaskLasso" + }, + { + "name": "fit", + "updates": [0] + } + ] + }, + "MultiTaskElasticNetCV": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "MultiTaskElasticNetCV" + }, + "_more_tags", + { + "name": "fit", + "updates": [0] + } + ] + }, + "MultiTaskLassoCV": { + "methods": [ + { + "name": "__init__", + "updates": [], + "returns": "MultiTaskLassoCV" + }, + "__init__", + "_more_tags", + { + "name": "fit", + "updates": [0] + } + ] + } + } + }, + "huber": { + "functions": ["_huber_loss_and_gradient"], + "types": { + "HuberRegressor": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "HuberRegressor" + }, + { + "name": "fit", + "updates": [0] + } + ] + } + } + }, + "least_angle": { + "functions": [ + "lars_path", + "lars_path_gram", + { + "name": "_lars_path_solver", + "updates": [3] + }, + "_check_copy_and_writeable", + { + "name": "_lars_path_residues", + "updates": [0] + } + ], + "types": { + "Lars": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "Lars" + }, + { + "name": "_fit", + "updates": [0] + }, + { + "name": "fit", + "updates": [0] + } + ] + }, + "LassoLars": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "LassoLars" + }, + { + "name": "fit", + "updates": [0] + } + ] + }, + "LarsCV": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "LarsCV" + }, + { + "name": "fit", + "updates": [0] + } + ] + }, + "LassoLarsCV": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "LassoLarsCV" + }, + { + "name": "fit", + "updates": [0] + } + ] + }, + "LassoLarsIC": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "LassoLarsIC" + }, + { + "name": "fit", + "updates": [0] + } + ] + } + } + }, + "logistic": { + "functions": [ + "_intercept_dot", + "_logistic_loss_and_grad", + "_logistic_loss", + "_logistic_grad_hess", + "_multinomial_loss", + "_multinomial_loss_grad", + "_multinomial_grad_hess", + "_check_solver", + "_check_multi_class", + "_logistic_regression_path", + "_log_reg_scoring_path" + ], + "types": { + "LogisticRegression": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "LogisticRegression" + }, + { + "name": "fit", + "updates": [0] + }, + "score", + "predict", + "predict_proba", + "predict_log_proba" + ] + }, + "LogisticRegressionCV": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "LogisticRegressionCV" + }, + { + "name": "fit", + "updates": [0] + }, + "score", + "predict", + "predict_proba", + "predict_log_proba" + ] + } + } + }, + "omp": { + "functions": [ + "_cholesky_omp", + { + "name": "_gram_omp", + "updates": [0, 1] + }, + "orthogonal_mp", + "orthogonal_mp_gram", + { + "name": "_omp_path_residues", + "updates": [0] + } + ], + "types": { + "OrthogonalMatchingPursuit": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "OrthogonalMatchingPursuit" + }, + { + "name": "fit", + "updates": [0] + } + ] + }, + "OrthogonalMatchingPursuitCV": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "OrthogonalMatchingPursuitCV" + }, + { + "name": "fit", + "updates": [0] + } + ] + } + } + }, + "passive_aggressive": { + "functions": [], + "types": { + "PassiveAggressiveClassifier": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "PassiveAggressiveClassifier" + }, + "partial_fit", + "fit" + ] + }, + "PassiveAggressiveRegressor": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "PassiveAggressiveRegressor" + }, + "partial_fit", + "fit" + ] + } + } + }, + "perceptron": { + "functions": [], + "types": { + "Perceptron": { + "methods": [ + { + "name": "__init__", + "updates": [], + "returns": "Perceptron" + }, + "__init__" + ] + } + } + }, + "ransac": { + "functions": ["_dynamic_max_trials"], + "types": { + "RANSACRegressor": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "RANSACRegressor" + }, + { + "name": "fit", + "updates": [0] + }, + "predict", + "score" + ] + } + } + }, + "ridge": { + "functions": [ + "_solve_sparse_cg", + "_solve_lsqr", + "_solve_cholesky", + "_solve_cholesky_kernel", + "_solve_svd", + "_get_valid_accept_sparse", + "ridge_regression", + "_ridge_regression", + "_check_gcv_mode", + "_find_smallest_angle" + ], + "types": { + "_BaseRidge": { + "methods": [ + { + "name": "fit", + "updates": [0] + } + ] + }, + "Ridge": { + "methods": [ + { + "name": "__init__", + "updates": [], + "returns": "Ridge" + }, + "__init__", + { + "name": "fit", + "updates": [0] + } + ] + }, + "RidgeClassifier": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "RidgeClassifier" + }, + { + "name": "fit", + "updates": [0] + }, + "predict", + "score" + ] + }, + "_X_operator": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "_X_operator" + }, + "_matvec", + "_matmat", + "_transpose" + ] + }, + "_Xt_operator": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "_Xt_operator" + }, + "_matvec", + "_matmat" + ] + }, + "_RidgeGCV": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "_RidgeGCV" + }, + "_decomp_diag", + "_diag_dot", + "_compute_gram", + "_compute_covariance", + "_sparse_multidot_diag", + "_eigen_decompose_gram", + "_solve_eigen_gram", + "_eigen_decompose_covariance", + "_solve_eigen_covariance_no_intercept", + "_solve_eigen_covariance_intercept", + "_solve_eigen_covariance", + "_svd_decompose_design_matrix", + "_solve_svd_design_matrix", + { + "name": "fit", + "updates": [0] + } + ] + }, + "_BaseRidgeCV": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "_BaseRidgeCV" + }, + { + "name": "fit", + "updates": [0] + } + ] + }, + "RidgeCV": { + "methods": [ + { + "name": "fit", + "updates": [0] + } + ] + }, + "RidgeClassifierCV": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "RidgeClassifierCV" + }, + { + "name": "fit", + "updates": [0] + }, + "predict", + "score" + ] + } + } + }, + "sag": { + "functions": ["get_auto_step_size", "sag_solver"], + "types": {} + }, + "setup": { + "functions": ["configuration"], + "types": {} + }, + "stochastic_gradient": { + "functions": ["_prepare_fit_binary", "fit_binary"], + "types": { + "_ValidationScoreCallback": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "_ValidationScoreCallback" + }, + "__call__" + ] + }, + "BaseSGD": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "BaseSGD" + }, + "set_params", + "_validate_params", + "_get_loss_function", + "_get_learning_rate_type", + "_get_penalty_type", + "_validate_sample_weight", + { + "name": "_allocate_parameter_mem", + "updates": [0] + }, + "_make_validation_split", + "_make_validation_score_cb" + ] + }, + "BaseSGDClassifier": { + "methods": [ + { + "name": "_partial_fit", + "updates": [0] + }, + { + "name": "_fit", + "updates": [0] + }, + { + "name": "_fit_binary", + "updates": [0] + }, + { + "name": "_fit_multiclass", + "updates": [0] + }, + { + "name": "partial_fit", + "updates": [0] + }, + { + "name": "fit", + "updates": [0] + }, + "set_params" + ] + }, + "SGDClassifier": { + "methods": [ + { + "name": "__init__", + "updates": [], + "returns": "SGDClassifier" + }, + "__init__", + "_check_proba", + "_predict_proba", + "_predict_log_proba", + { + "name": "partial_fit", + "updates": [0] + }, + { + "name": "fit", + "updates": [0] + }, + "set_params" + ] + }, + "BaseSGDRegressor": { + "methods": [ + { + "name": "_partial_fit", + "updates": [0] + }, + { + "name": "partial_fit", + "updates": [0] + }, + { + "name": "_fit", + "updates": [0] + }, + { + "name": "fit", + "updates": [0] + }, + "_decision_function", + "predict", + { + "name": "_fit_regressor", + "updates": [0] + }, + "set_params" + ] + }, + "SGDRegressor": { + "methods": [ + { + "name": "__init__", + "updates": [], + "returns": "SGDRegressor" + }, + "__init__", + { + "name": "partial_fit", + "updates": [0] + }, + { + "name": "fit", + "updates": [0] + }, + "predict", + "set_params" + ] + } + } + }, + "theil_sen": { + "functions": [ + "_modified_weiszfeld_step", + "_spatial_median", + "_breakdown_point", + "_lstsq" + ], + "types": { + "TheilSenRegressor": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "TheilSenRegressor" + }, + "_check_subparams", + { + "name": "fit", + "updates": [0] + } + ] + } + } + } + }, + "types": { + "LinearRegression": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "LinearRegression" + }, + { + "name": "fit", + "updates": [0] + }, + "predict" + ] + }, + "BayesianRidge": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "BayesianRidge" + }, + { + "name": "fit", + "updates": [0] + }, + "predict", + "_update_coef_", + "_log_marginal_likelihood" + ] + }, + "ARDRegression": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "ARDRegression" + }, + { + "name": "fit", + "updates": [0] + }, + "predict" + ] + }, + "Lars": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "Lars" + }, + { + "name": "_fit", + "updates": [0] + }, + { + "name": "fit", + "updates": [0] + } + ] + }, + "LassoLars": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "LassoLars" + }, + { + "name": "fit", + "updates": [0] + } + ] + }, + "LarsCV": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "LarsCV" + }, + { + "name": "fit", + "updates": [0] + } + ] + }, + "LassoLarsCV": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "LassoLarsCV" + }, + { + "name": "fit", + "updates": [0] + } + ] + }, + "LassoLarsIC": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "LassoLarsIC" + }, + { + "name": "fit", + "updates": [0] + } + ] + }, + "Lasso": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "Lasso" + }, + "__init__", + { + "name": "fit", + "updates": [0] + } + ] + }, + "ElasticNet": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "ElasticNet" + }, + { + "name": "fit", + "updates": [0] + }, + "_decision_function" + ] + }, + "LassoCV": { + "methods": [ + { + "name": "__init__", + "updates": [], + "returns": "LassoCV" + }, + "__init__", + { + "name": "fit", + "updates": [0] + } + ] + }, + "ElasticNetCV": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "ElasticNetCV" + }, + { + "name": "fit", + "updates": [0] + } + ] + }, + "MultiTaskLasso": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "MultiTaskLasso" + }, + { + "name": "fit", + "updates": [0] + } + ] + }, + "MultiTaskElasticNet": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "MultiTaskElasticNet" + }, + { + "name": "fit", + "updates": [0] + }, + "_more_tags" + ] + }, + "MultiTaskElasticNetCV": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "MultiTaskElasticNetCV" + }, + "_more_tags", + { + "name": "fit", + "updates": [0] + } + ] + }, + "MultiTaskLassoCV": { + "methods": [ + { + "name": "__init__", + "updates": [], + "returns": "MultiTaskLassoCV" + }, + "__init__", + "_more_tags", + { + "name": "fit", + "updates": [0] + } + ] + }, + "HuberRegressor": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "HuberRegressor" + }, + { + "name": "fit", + "updates": [0] + } + ] + }, + "SGDClassifier": { + "methods": [ + { + "name": "__init__", + "updates": [], + "returns": "SGDClassifier" + }, + "__init__", + "_check_proba", + "_predict_proba", + "_predict_log_proba", + { + "name": "partial_fit", + "updates": [0] + }, + { + "name": "fit", + "updates": [0] + }, + "set_params" + ] + }, + "SGDRegressor": { + "methods": [ + { + "name": "__init__", + "updates": [], + "returns": "SGDRegressor" + }, + "__init__", + { + "name": "partial_fit", + "updates": [0] + }, + { + "name": "fit", + "updates": [0] + }, + "predict", + "set_params" + ] + }, + "Ridge": { + "methods": [ + { + "name": "__init__", + "updates": [], + "returns": "Ridge" + }, + "__init__", + { + "name": "fit", + "updates": [0] + } + ] + }, + "RidgeCV": { + "methods": [ + { + "name": "fit", + "updates": [0] + } + ] + }, + "RidgeClassifier": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "RidgeClassifier" + }, + { + "name": "fit", + "updates": [0] + } + ] + }, + "RidgeClassifierCV": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "RidgeClassifierCV" + }, + { + "name": "fit", + "updates": [0] + } + ] + }, + "LogisticRegression": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "LogisticRegression" + }, + { + "name": "fit", + "updates": [0] + }, + "score", + "predict", + "predict_proba", + "predict_log_proba" + ] + }, + "LogisticRegressionCV": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "LogisticRegressionCV" + }, + { + "name": "fit", + "updates": [0] + }, + "score", + "predict", + "predict_proba", + "predict_log_proba" + ] + }, + "OrthogonalMatchingPursuit": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "OrthogonalMatchingPursuit" + }, + { + "name": "fit", + "updates": [0] + } + ] + }, + "OrthogonalMatchingPursuitCV": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "OrthogonalMatchingPursuitCV" + }, + { + "name": "fit", + "updates": [0] + } + ] + }, + "PassiveAggressiveClassifier": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "PassiveAggressiveClassifier" + }, + { "name": "partial_fit", "updates": [0] }, + { "name": "fit", "updates": [0] }, + "score", + "predict" + ] + }, + "PassiveAggressiveRegressor": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "PassiveAggressiveRegressor" + }, + { "name": "partial_fit", "updates": [0] }, + { "name": "fit", "updates": [0] }, + "score", + "predict" + ] + }, + "Perceptron": { + "methods": [ + { + "name": "__init__", + "updates": [], + "returns": "Perceptron" + }, + { "name": "partial_fit", "updates": [0] }, + { "name": "fit", "updates": [0] }, + "score", + "predict" + ] + }, + "RANSACRegressor": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "RANSACRegressor" + }, + { + "name": "fit", + "updates": [0] + }, + "predict", + "score" + ] + }, + "TheilSenRegressor": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "TheilSenRegressor" + }, + "_check_subparams", + { + "name": "fit", + "updates": [0] + } + ] + } + }, + "functions": [ + "lars_path", + "lars_path_gram", + "lasso_path", + "enet_path", + "ridge_regression", + "orthogonal_mp", + "orthogonal_mp_gram" + ] + }, + "manifold": { + "modules": { + "tests": { + "modules": { + "test_isomap": { + "functions": [ + "test_isomap_simple_grid", + "test_isomap_reconstruction_error", + "test_transform", + "test_pipeline", + "test_isomap_clone_bug", + "test_sparse_input" + ], + "types": {} + }, + "test_locally_linear": { + "functions": [ + "test_barycenter_kneighbors_graph", + "test_lle_simple_grid", + "test_lle_manifold", + "test_lle_init_parameters", + "test_pipeline", + "test_singular_matrix", + "test_integer_input" + ], + "types": {} + }, + "test_mds": { + "functions": ["test_smacof", "test_smacof_error", "test_MDS"], + "types": {} + }, + "test_spectral_embedding": { + "functions": [ + "_check_with_col_sign_flipping", + "test_sparse_graph_connected_component", + "test_spectral_embedding_precomputed_affinity", + "test_spectral_embedding_callable_affinity", + "test_spectral_embedding_amg_solver", + "test_spectral_embedding_unknown_eigensolver", + "test_spectral_embedding_unknown_affinity", + "test_connectivity", + "test_spectral_embedding_deterministic", + "test_spectral_embedding_unnormalized", + "test_spectral_embedding_first_eigen_vector" + ], + "types": {} + }, + "test_t_sne": { + "functions": [ + "test_gradient_descent_stops", + "test_binary_search", + "test_binary_search_neighbors", + "test_binary_perplexity_stability", + "test_gradient", + "test_trustworthiness", + "test_preserve_trustworthiness_approximately", + "test_optimization_minimizes_kl_divergence", + "test_fit_csr_matrix", + "test_preserve_trustworthiness_approximately_with_precomputed_distances", + "test_trustworthiness_precomputed_deprecation", + "test_trustworthiness_not_euclidean_metric", + "test_early_exaggeration_too_small", + "test_too_few_iterations", + "test_non_square_precomputed_distances", + "test_non_positive_precomputed_distances", + "test_non_positive_computed_distances", + "test_init_not_available", + "test_init_ndarray", + "test_init_ndarray_precomputed", + "test_distance_not_available", + "test_method_not_available", + "test_angle_out_of_range_checks", + "test_pca_initialization_not_compatible_with_precomputed_kernel", + "test_n_components_range", + "test_early_exaggeration_used", + "test_n_iter_used", + "test_answer_gradient_two_points", + "test_answer_gradient_four_points", + "test_skip_num_points_gradient", + "_run_answer_test", + "test_verbose", + "test_chebyshev_metric", + "test_reduction_to_one_component", + "test_no_sparse_on_barnes_hut", + "test_barnes_hut_angle", + "test_min_grad_norm", + "test_accessible_kl_divergence", + "check_uniform_grid", + "assert_uniform_grid", + "test_bh_match_exact", + "test_tsne_with_different_distance_metrics" + ], + "types": { + "ObjectiveSmallGradient": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "ObjectiveSmallGradient" + }, + { + "name": "__call__", + "updates": [0] + } + ] + } + } + } + }, + "types": {}, + "functions": [] + }, + "isomap": { + "functions": [], + "types": { + "Isomap": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "Isomap" + }, + { + "name": "_fit_transform", + "updates": [0] + }, + "reconstruction_error", + { + "name": "fit", + "updates": [0] + }, + { + "name": "fit_transform", + "updates": [0] + }, + "transform" + ] + } + } + }, + "locally_linear": { + "functions": [ + "barycenter_weights", + "barycenter_kneighbors_graph", + "null_space", + "locally_linear_embedding" + ], + "types": { + "LocallyLinearEmbedding": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "LocallyLinearEmbedding" + }, + { + "name": "_fit_transform", + "updates": [0] + }, + { + "name": "fit", + "updates": [0] + }, + { + "name": "fit_transform", + "updates": [0] + }, + "transform" + ] + } + } + }, + "mds": { + "functions": ["_smacof_single", "smacof"], + "types": { + "MDS": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "MDS" + }, + { + "name": "fit", + "updates": [0] + }, + { + "name": "fit_transform", + "updates": [0] + } + ] + } + } + }, + "setup": { + "functions": ["configuration"], + "types": {} + }, + "spectral_embedding_": { + "functions": [ + "_graph_connected_component", + "_graph_is_connected", + "_set_diag", + "spectral_embedding" + ], + "types": { + "SpectralEmbedding": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "SpectralEmbedding" + }, + { + "name": "_get_affinity_matrix", + "updates": [0] + }, + { + "name": "fit", + "updates": [0] + }, + { + "name": "fit_transform", + "updates": [0] + } + ] + } + } + }, + "t_sne": { + "functions": [ + "_joint_probabilities", + "_joint_probabilities_nn", + "_kl_divergence", + "_kl_divergence_bh", + { + "name": "_gradient_descent", + "updates": [12] + }, + "trustworthiness" + ], + "types": { + "TSNE": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "TSNE" + }, + { + "name": "_fit", + "updates": [0] + }, + { + "name": "_tsne", + "updates": [0] + }, + { + "name": "fit_transform", + "updates": [0] + }, + { + "name": "fit", + "updates": [0] + } + ] + } + } + } + }, + "types": { + "LocallyLinearEmbedding": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "LocallyLinearEmbedding" + }, + { + "name": "_fit_transform", + "updates": [0] + }, + { + "name": "fit", + "updates": [0] + }, + { + "name": "fit_transform", + "updates": [0] + }, + "transform" + ] + }, + "Isomap": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "Isomap" + }, + { + "name": "_fit_transform", + "updates": [0] + }, + "reconstruction_error", + { + "name": "fit", + "updates": [0] + }, + { + "name": "fit_transform", + "updates": [0] + }, + "transform" + ] + }, + "MDS": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "MDS" + }, + { + "name": "fit", + "updates": [0] + }, + { + "name": "fit_transform", + "updates": [0] + } + ] + }, + "SpectralEmbedding": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "SpectralEmbedding" + }, + { + "name": "_get_affinity_matrix", + "updates": [0] + }, + { + "name": "fit", + "updates": [0] + }, + { + "name": "fit_transform", + "updates": [0] + } + ] + }, + "TSNE": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "TSNE" + }, + { + "name": "_fit", + "updates": [0] + }, + { + "name": "_tsne", + "updates": [0] + }, + { + "name": "fit_transform", + "updates": [0] + }, + { + "name": "fit", + "updates": [0] + } + ] + } + }, + "functions": [ + "locally_linear_embedding", + "smacof", + "spectral_embedding" + ] + }, + "metrics": { + "modules": { + "cluster": { + "modules": { + "tests": { + "modules": { + "test_bicluster": { + "functions": [ + "test_jaccard", + "test_consensus_score", + "test_consensus_score_issue2445" + ], + "types": {} + }, + "test_common": { + "functions": ["test_symmetric_non_symmetric_union"], + "types": {} + }, + "test_supervised": { + "functions": [ + "test_future_warning", + "test_generalized_average", + "test_homogeneous_but_not_complete_labeling", + "test_complete_but_not_homogeneous_labeling", + "test_not_complete_and_not_homogeneous_labeling", + "test_beta_parameter", + "test_non_consecutive_labels", + "test_expected_mutual_info_overflow", + "test_int_overflow_mutual_info_fowlkes_mallows_score", + "test_entropy", + "test_contingency_matrix", + "test_contingency_matrix_sparse", + "test_v_measure_and_mutual_information", + "test_fowlkes_mallows_score", + "test_fowlkes_mallows_score_properties" + ], + "types": {} + }, + "test_unsupervised": { + "functions": [ + "test_silhouette", + "test_cluster_size_1", + "test_silhouette_paper_example", + "test_correct_labelsize", + "test_non_encoded_labels", + "test_non_numpy_labels", + "assert_raises_on_only_one_label", + "assert_raises_on_all_points_same_cluster", + "test_calinski_harabasz_score", + "test_deprecated_calinski_harabaz_score", + "test_davies_bouldin_score" + ], + "types": {} + } + }, + "types": {}, + "functions": [] + }, + "bicluster": { + "functions": [ + "_check_rows_and_columns", + "_jaccard", + "_pairwise_similarity", + "consensus_score" + ], + "types": {} + }, + "setup": { + "functions": ["configuration"], + "types": {} + }, + "supervised": { + "functions": [ + "_comb2", + "check_clusterings", + "_generalized_average", + "contingency_matrix", + "adjusted_rand_score", + "homogeneity_completeness_v_measure", + "homogeneity_score", + "completeness_score", + "v_measure_score", + "mutual_info_score", + "adjusted_mutual_info_score", + "normalized_mutual_info_score", + "fowlkes_mallows_score", + "entropy" + ], + "types": {} + }, + "unsupervised": { + "functions": [ + "check_number_of_labels", + "silhouette_score", + "_silhouette_reduce", + "silhouette_samples", + "calinski_harabasz_score", + "davies_bouldin_score" + ], + "types": {} + } + }, + "types": {}, + "functions": [ + "adjusted_mutual_info_score", + "normalized_mutual_info_score", + "adjusted_rand_score", + "completeness_score", + "contingency_matrix", + "homogeneity_completeness_v_measure", + "homogeneity_score", + "mutual_info_score", + "v_measure_score", + "fowlkes_mallows_score", + "entropy", + "silhouette_samples", + "silhouette_score", + "calinski_harabasz_score", + "davies_bouldin_score", + "consensus_score" + ] + }, + "tests": { + "modules": { + "test_classification": { + "functions": [ + "make_prediction", + "test_classification_report_dictionary_output", + "test_multilabel_accuracy_score_subset_accuracy", + "test_precision_recall_f1_score_binary", + "test_average_precision_score_score_non_binary_class", + "test_average_precision_score_duplicate_values", + "test_average_precision_score_tied_values", + "test_precision_recall_f_unused_pos_label", + "test_confusion_matrix_binary", + "test_multilabel_confusion_matrix_binary", + "test_multilabel_confusion_matrix_multiclass", + "test_multilabel_confusion_matrix_multilabel", + "test_multilabel_confusion_matrix_errors", + "test_cohen_kappa", + "test_matthews_corrcoef_against_numpy_corrcoef", + "test_matthews_corrcoef_against_jurman", + "test_matthews_corrcoef", + "test_matthews_corrcoef_multiclass", + "test_precision_recall_f1_score_multiclass", + "test_precision_recall_f1_score_binary_averaged", + "test_zero_precision_recall", + "test_confusion_matrix_multiclass_subset_labels", + "test_confusion_matrix_dtype", + "test_classification_report_multiclass", + "test_classification_report_multiclass_balanced", + "test_classification_report_multiclass_with_label_detection", + "test_classification_report_multiclass_with_digits", + "test_classification_report_multiclass_with_string_label", + "test_classification_report_multiclass_with_unicode_label", + "test_classification_report_multiclass_with_long_string_label", + "test_classification_report_labels_target_names_unequal_length", + "test_classification_report_no_labels_target_names_unequal_length", + "test_multilabel_zero_one_loss_subset", + "test_multilabel_hamming_loss", + "test_jaccard_score_validation", + "test_multilabel_jaccard_score", + "test_multiclass_jaccard_score", + "test_average_binary_jaccard_score", + "test_precision_recall_f1_no_labels_average_none", + "test_prf_warnings", + "test_recall_warnings", + "test_precision_warnings", + "test_fscore_warnings", + "test_prf_average_binary_data_non_binary", + "test__check_targets", + "test__check_targets_multiclass_with_both_y_true_and_y_pred_binary", + "test_hinge_loss_binary", + "test_hinge_loss_multiclass", + "test_hinge_loss_multiclass_missing_labels_with_labels_none", + "test_hinge_loss_multiclass_with_missing_labels", + "test_hinge_loss_multiclass_invariance_lists", + "test_log_loss", + "test_log_loss_pandas_input", + "test_brier_score_loss", + "test_balanced_accuracy_score_unseen", + "test_multilabel_jaccard_similarity_score_deprecation" + ], + "types": {} + }, + "test_common": { + "functions": [ + "precision_recall_curve_padded_thresholds", + "test_normalize_option_multilabel_classification", + "check_averaging", + "test_averaging_binary_multilabel_all_zeroes" + ], + "types": {} + }, + "test_pairwise": { + "functions": [ + "test_pairwise_distances", + "test_no_data_conversion_warning", + "test_pairwise_precomputed_non_negative", + "callable_rbf_kernel", + "test_pairwise_callable_nonstrict_metric", + "test_pairwise_kernels_callable", + "test_pairwise_kernels_filter_param", + "test_paired_distances_callable", + "test_pairwise_distances_argmin_min", + "_reduce_func", + "test_pairwise_distances_chunked_reduce", + "check_pairwise_distances_chunked", + "test_cosine_distances", + "test_haversine_distances", + "test_paired_euclidean_distances", + "test_paired_manhattan_distances", + "test_chi_square_kernel", + "test_linear_kernel", + "test_rbf_kernel", + "test_laplacian_kernel", + "test_cosine_similarity", + "test_check_dense_matrices", + "test_check_XB_returned", + "test_check_different_dimensions", + "test_check_invalid_dimensions", + "test_check_sparse_arrays", + "tuplify", + "test_check_tuple_input", + "test_check_preserve_type" + ], + "types": {} + }, + "test_ranking": { + "functions": [ + "make_prediction", + "_auc", + "_average_precision", + "_average_precision_slow", + "_partial_roc_auc_score", + "test_roc_curve_end_points", + "test_roc_returns_consistency", + "test_roc_curve_multi", + "test_roc_curve_confidence", + "test_roc_curve_hard", + "test_roc_curve_one_label", + "test_roc_curve_toydata", + "test_roc_curve_drop_intermediate", + "test_roc_curve_fpr_tpr_increasing", + "test_auc", + "test_auc_errors", + "test_deprecated_auc_reorder", + "test_auc_score_non_binary_class", + "test_binary_clf_curve", + "test_precision_recall_curve", + "_test_precision_recall_curve", + "test_precision_recall_curve_errors", + "test_precision_recall_curve_toydata", + "test_average_precision_constant_values", + "test_average_precision_score_pos_label_errors", + "test_score_scale_invariance", + "check_lrap_toy", + "check_zero_or_all_relevant_labels", + "check_lrap_error_raised", + "check_lrap_only_ties", + "check_lrap_without_tie_and_increasing_score", + "_my_lrap", + "check_alternative_lrap_implementation", + "test_lrap_error_raised", + "test_lrap_sample_weighting_zero_labels", + "test_coverage_error", + "test_coverage_tie_handling", + "test_label_ranking_loss", + "test_ranking_appropriate_input_shape", + "test_ranking_loss_ties_handling", + "test_partial_roc_auc_score" + ], + "types": {} + }, + "test_regression": { + "functions": [ + "test_regression_metrics", + "test_multioutput_regression", + "test_regression_metrics_at_limits", + "test__check_reg_targets", + "test__check_reg_targets_exception", + "test_regression_multioutput_array", + "test_regression_custom_weights" + ], + "types": {} + }, + "test_score_objects": { + "functions": [ + "_make_estimators", + "setup_module", + "teardown_module", + "test_all_scorers_repr", + "check_scoring_validator_for_single_metric_usecases", + "check_multimetric_scoring_single_metric_wrapper", + "test_check_scoring_and_check_multimetric_scoring", + "test_make_scorer", + "test_classification_scores", + "test_regression_scorers", + "test_thresholded_scorers_multilabel_indicator_data" + ], + "types": { + "EstimatorWithoutFit": { + "methods": [] + }, + "EstimatorWithFit": { + "methods": ["fit"] + }, + "EstimatorWithFitAndScore": { + "methods": ["fit", "score"] + }, + "EstimatorWithFitAndPredict": { + "methods": [ + { + "name": "fit", + "updates": [0] + }, + "predict" + ] + }, + "DummyScorer": { + "methods": ["__call__"] + } + } + } + }, + "types": {}, + "functions": [] + }, + "base": { + "functions": ["_average_binary_score"], + "types": {} + }, + "classification": { + "functions": [ + "_check_targets", + "_weighted_sum", + "accuracy_score", + "confusion_matrix", + "multilabel_confusion_matrix", + "cohen_kappa_score", + "jaccard_similarity_score", + "jaccard_score", + "matthews_corrcoef", + "zero_one_loss", + "f1_score", + "fbeta_score", + { + "name": "_prf_divide", + "updates": [1] + }, + "_check_set_wise_labels", + "precision_recall_fscore_support", + "precision_score", + "recall_score", + "balanced_accuracy_score", + "classification_report", + "hamming_loss", + "log_loss", + "hinge_loss", + "brier_score_loss" + ], + "types": {} + }, + "pairwise": { + "functions": [ + "_return_float_dtype", + "check_pairwise_arrays", + "check_paired_arrays", + "euclidean_distances", + "_euclidean_distances_upcast", + "_argmin_min_reduce", + "pairwise_distances_argmin_min", + "pairwise_distances_argmin", + "haversine_distances", + "manhattan_distances", + "cosine_distances", + "paired_euclidean_distances", + "paired_manhattan_distances", + "paired_cosine_distances", + "paired_distances", + "linear_kernel", + "polynomial_kernel", + "sigmoid_kernel", + "rbf_kernel", + "laplacian_kernel", + "cosine_similarity", + "additive_chi2_kernel", + "chi2_kernel", + "distance_metrics", + { + "name": "_dist_wrapper", + "updates": [1] + }, + "_parallel_pairwise", + "_pairwise_callable", + "_check_chunk_size", + "_precompute_metric_params", + "pairwise_distances_chunked", + "pairwise_distances", + "kernel_metrics", + "pairwise_kernels" + ], + "types": {} + }, + "ranking": { + "functions": [ + "auc", + "average_precision_score", + "roc_auc_score", + "_binary_clf_curve", + "precision_recall_curve", + "roc_curve", + "label_ranking_average_precision_score", + "coverage_error", + "label_ranking_loss" + ], + "types": {} + }, + "regression": { + "functions": [ + "_check_reg_targets", + "mean_absolute_error", + "mean_squared_error", + "mean_squared_log_error", + "median_absolute_error", + "explained_variance_score", + "r2_score", + "max_error" + ], + "types": {} + }, + "scorer": { + "functions": [ + "get_scorer", + "_passthrough_scorer", + "check_scoring", + "_check_multimetric_scoring", + "make_scorer" + ], + "types": { + "_BaseScorer": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "_BaseScorer" + }, + "__repr__", + "_factory_args" + ] + }, + "_PredictScorer": { + "methods": ["__call__"] + }, + "_ProbaScorer": { + "methods": ["__call__", "_factory_args"] + }, + "_ThresholdScorer": { + "methods": ["__call__", "_factory_args"] + } + } + }, + "setup": { + "functions": ["configuration"], + "types": {} + } + }, + "types": {}, + "functions": [ + "auc", + "average_precision_score", + "coverage_error", + "label_ranking_average_precision_score", + "label_ranking_loss", + "precision_recall_curve", + "roc_auc_score", + "roc_curve", + "accuracy_score", + "balanced_accuracy_score", + "classification_report", + "cohen_kappa_score", + "confusion_matrix", + "f1_score", + "fbeta_score", + "hamming_loss", + "hinge_loss", + "jaccard_similarity_score", + "jaccard_score", + "log_loss", + "matthews_corrcoef", + "precision_recall_fscore_support", + "precision_score", + "recall_score", + "zero_one_loss", + "brier_score_loss", + "multilabel_confusion_matrix", + "adjusted_mutual_info_score", + "adjusted_rand_score", + "completeness_score", + "consensus_score", + "homogeneity_completeness_v_measure", + "homogeneity_score", + "mutual_info_score", + "normalized_mutual_info_score", + "fowlkes_mallows_score", + "silhouette_samples", + "silhouette_score", + "calinski_harabasz_score", + "v_measure_score", + "davies_bouldin_score", + "euclidean_distances", + "pairwise_distances", + "pairwise_distances_argmin", + "pairwise_distances_argmin_min", + "pairwise_kernels", + "pairwise_distances_chunked", + "explained_variance_score", + "max_error", + "mean_absolute_error", + "mean_squared_error", + "mean_squared_log_error", + "median_absolute_error", + "r2_score", + "check_scoring", + "make_scorer", + "get_scorer" + ] + }, + "mixture": { + "modules": { + "tests": { + "modules": { + "test_bayesian_mixture": { + "functions": [ + "test_log_dirichlet_norm", + "test_log_wishart_norm", + "test_bayesian_mixture_covariance_type", + "test_bayesian_mixture_weight_concentration_prior_type", + "test_bayesian_mixture_weights_prior_initialisation", + "test_bayesian_mixture_mean_prior_initialisation", + "test_bayesian_mixture_precisions_prior_initialisation", + "test_bayesian_mixture_check_is_fitted", + "test_bayesian_mixture_weights", + "test_compare_covar_type", + "test_bayesian_mixture_fit_predict_n_init", + "test_bayesian_mixture_predict_predict_proba" + ], + "types": {} + }, + "test_gaussian_mixture": { + "functions": [ + "generate_data", + "test_gaussian_mixture_attributes", + "test_check_X", + "test_check_weights", + "test_check_means", + "test_check_precisions", + "test_suffstat_sk_full", + "test_suffstat_sk_tied", + "test_suffstat_sk_diag", + "test_gaussian_suffstat_sk_spherical", + "test_compute_log_det_cholesky", + "_naive_lmvnpdf_diag", + "test_gaussian_mixture_log_probabilities", + "test_gaussian_mixture_estimate_log_prob_resp", + "test_gaussian_mixture_predict_predict_proba", + "test_gaussian_mixture_fit_predict_n_init", + "test_gaussian_mixture_fit", + "test_gaussian_mixture_fit_best_params", + "test_gaussian_mixture_fit_convergence_warning", + "test_multiple_init", + "test_gaussian_mixture_n_parameters", + "test_bic_1d_1component", + "test_gaussian_mixture_aic_bic", + "test_gaussian_mixture_verbose", + "test_score", + "test_score_samples", + "test_monotonic_likelihood", + "test_regularisation", + "test_property", + "test_sample" + ], + "types": { + "RandomData": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "RandomData" + } + ] + } + } + }, + "test_mixture": { + "functions": [], + "types": {} + } + }, + "types": {}, + "functions": [] + }, + "base": { + "functions": ["_check_shape", "_check_X"], + "types": { + "BaseMixture": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "BaseMixture" + }, + "_check_initial_parameters", + "_initialize_parameters", + { + "name": "fit", + "updates": [0] + }, + { + "name": "fit_predict", + "updates": [0] + }, + "_e_step", + "score_samples", + "score", + "predict", + "predict_proba", + "sample", + "_estimate_weighted_log_prob", + "_estimate_log_prob_resp", + { + "name": "_print_verbose_msg_init_beg", + "updates": [0] + }, + { + "name": "_print_verbose_msg_iter_end", + "updates": [0] + }, + "_print_verbose_msg_init_end" + ] + } + } + }, + "bayesian_mixture": { + "functions": ["_log_dirichlet_norm", "_log_wishart_norm"], + "types": { + "BayesianGaussianMixture": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "BayesianGaussianMixture" + }, + { + "name": "_check_parameters", + "updates": [0] + }, + { + "name": "_check_weights_parameters", + "updates": [0] + }, + { + "name": "_check_means_parameters", + "updates": [0] + }, + { + "name": "_check_precision_parameters", + "updates": [0] + }, + { + "name": "_checkcovariance_prior_parameter", + "updates": [0] + }, + { + "name": "_initialize", + "updates": [0] + }, + { + "name": "_estimate_weights", + "updates": [0] + }, + { + "name": "_estimate_means", + "updates": [0] + }, + { + "name": "_estimate_precisions", + "updates": [0] + }, + { + "name": "_estimate_wishart_full", + "updates": [0] + }, + { + "name": "_estimate_wishart_tied", + "updates": [0] + }, + { + "name": "_estimate_wishart_diag", + "updates": [0] + }, + { + "name": "_estimate_wishart_spherical", + "updates": [0] + }, + "_check_is_fitted", + { + "name": "_m_step", + "updates": [0] + }, + "_estimate_log_weights", + "_estimate_log_prob", + "_compute_lower_bound", + "_get_parameters", + { + "name": "_set_parameters", + "updates": [0] + } + ] + } + } + }, + "gaussian_mixture": { + "functions": [ + "_check_weights", + "_check_means", + "_check_precision_positivity", + "_check_precision_matrix", + "_check_precisions_full", + "_check_precisions", + "_estimate_gaussian_covariances_full", + "_estimate_gaussian_covariances_tied", + "_estimate_gaussian_covariances_diag", + "_estimate_gaussian_covariances_spherical", + "_estimate_gaussian_parameters", + "_compute_precision_cholesky", + "_compute_log_det_cholesky", + "_estimate_log_gaussian_prob" + ], + "types": { + "GaussianMixture": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "GaussianMixture" + }, + { + "name": "_check_parameters", + "updates": [0] + }, + { + "name": "_initialize", + "updates": [0] + }, + { + "name": "_m_step", + "updates": [0] + }, + "_estimate_log_prob", + "_estimate_log_weights", + "_compute_lower_bound", + "_check_is_fitted", + "_get_parameters", + { + "name": "_set_parameters", + "updates": [0] + }, + "_n_parameters", + "bic", + "aic" + ] + } + } + } + }, + "types": { + "GaussianMixture": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "GaussianMixture" + }, + { + "name": "_check_parameters", + "updates": [0] + }, + { + "name": "_initialize", + "updates": [0] + }, + { + "name": "_m_step", + "updates": [0] + }, + "_estimate_log_prob", + "_estimate_log_weights", + "_compute_lower_bound", + "_check_is_fitted", + "_get_parameters", + { + "name": "_set_parameters", + "updates": [0] + }, + "_n_parameters", + "bic", + "aic" + ] + }, + "BayesianGaussianMixture": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "BayesianGaussianMixture" + }, + { + "name": "_check_parameters", + "updates": [0] + }, + { + "name": "_check_weights_parameters", + "updates": [0] + }, + { + "name": "_check_means_parameters", + "updates": [0] + }, + { + "name": "_check_precision_parameters", + "updates": [0] + }, + { + "name": "_checkcovariance_prior_parameter", + "updates": [0] + }, + { + "name": "_initialize", + "updates": [0] + }, + { + "name": "_estimate_weights", + "updates": [0] + }, + { + "name": "_estimate_means", + "updates": [0] + }, + { + "name": "_estimate_precisions", + "updates": [0] + }, + { + "name": "_estimate_wishart_full", + "updates": [0] + }, + { + "name": "_estimate_wishart_tied", + "updates": [0] + }, + { + "name": "_estimate_wishart_diag", + "updates": [0] + }, + { + "name": "_estimate_wishart_spherical", + "updates": [0] + }, + "_check_is_fitted", + { + "name": "_m_step", + "updates": [0] + }, + "_estimate_log_weights", + "_estimate_log_prob", + "_compute_lower_bound", + "_get_parameters", + { + "name": "_set_parameters", + "updates": [0] + } + ] + } + }, + "functions": [] + }, + "model_selection": { + "modules": { + "tests": { + "modules": { + "common": { + "functions": [], + "types": { + "OneTimeSplitter": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "OneTimeSplitter" + }, + "split", + "get_n_splits" + ] + } + } + }, + "test_search": { + "functions": [ + "assert_grid_iter_equals_getitem", + "test_parameter_grid", + "check_hyperparameter_searcher_with_fit_params", + "test_refit_callable", + "test_refit_callable_invalid_type", + "test_refit_callable_multi_metric", + "test_param_sampler", + "check_cv_results_array_types", + "check_cv_results_keys", + "test_grid_search_cv_results", + "test_random_search_cv_results", + "test_grid_search_cv_results_multimetric", + "test_random_search_cv_results_multimetric", + "compare_cv_results_multimetric_with_single", + "compare_refit_methods_when_refit_with_acc", + "test_parameters_sampler_replacement", + "test_custom_run_search", + "test__custom_fit_no_run_search", + "test_deprecated_grid_search_iid", + "test_empty_cv_iterator_error", + "test_random_search_bad_cv" + ], + "types": { + "MockClassifier": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "MockClassifier" + }, + { + "name": "fit", + "updates": [0] + }, + "predict", + "transform", + "inverse_transform", + "score", + "get_params", + { + "name": "set_params", + "updates": [0] + } + ] + }, + "LinearSVCNoScore": { + "methods": [] + }, + "BrokenClassifier": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "BrokenClassifier" + }, + { + "name": "fit", + "updates": [0] + }, + "predict" + ] + }, + "FailingClassifier": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "FailingClassifier" + }, + "fit", + "predict", + "score" + ] + }, + "CustomSearchCV": { + "methods": [ + { + "name": "__init__", + "updates": [], + "returns": "CustomSearchCV" + }, + "__init__", + "_run_search" + ] + }, + "NoRunSearchSearchCV": { + "methods": [ + { + "name": "__init__", + "updates": [], + "returns": "NoRunSearchSearchCV" + }, + "__init__", + "fit" + ] + }, + "BadSearchCV": { + "methods": [ + { + "name": "__init__", + "updates": [], + "returns": "BadSearchCV" + }, + "__init__" + ] + }, + "BrokenKFold": { + "methods": ["get_n_splits"] + } + } + }, + "test_split": { + "functions": [ + "check_valid_split", + "check_cv_coverage", + "test_kfold_valueerrors", + "test_kfold_indices", + "test_kfold_no_shuffle", + "test_stratified_kfold_no_shuffle", + "test_stratified_kfold_ratios", + "test_kfold_balance", + "test_stratifiedkfold_balance", + "test_shuffle_kfold", + "test_shuffle_kfold_stratifiedkfold_reproducibility", + "test_shuffle_stratifiedkfold", + "test_kfold_can_detect_dependent_samples_on_digits", + "test_shuffle_split", + "test_stratified_shuffle_split_respects_test_size", + "test_stratified_shuffle_split_iter", + "test_stratified_shuffle_split_even", + "test_stratified_shuffle_split_overlap_train_test_bug", + "test_stratified_shuffle_split_multilabel", + "test_stratified_shuffle_split_multilabel_many_labels", + "test_predefinedsplit_with_kfold_split", + "test_group_shuffle_split", + "test_leave_one_p_group_out", + "test_leave_group_out_changing_groups", + "test_leave_one_p_group_out_error_on_fewer_number_of_groups", + "test_repeated_kfold_determinstic_split", + "test_get_n_splits_for_repeated_kfold", + "test_get_n_splits_for_repeated_stratified_kfold", + "test_repeated_stratified_kfold_determinstic_split", + "test_train_test_split_errors", + "test_train_test_split", + "test_train_test_split_sparse", + "test_train_test_split_mock_pandas", + "test_train_test_split_list_input", + "test_shufflesplit_reproducible", + "test_stratifiedshufflesplit_list_input", + "test_train_test_split_allow_nans", + "test_check_cv", + "test_cv_iterable_wrapper", + "test_group_kfold", + "test_time_series_cv", + "_check_time_series_max_train_size", + "test_time_series_max_train_size", + "test_nsplit_default_warn", + "test_check_cv_default_warn", + "test_build_repr", + "test_train_test_split_empty_trainset", + "test_leave_one_out_empty_trainset", + "test_leave_p_out_empty_trainset" + ], + "types": { + "MockClassifier": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "MockClassifier" + }, + { + "name": "fit", + "updates": [0] + }, + "predict", + "score", + "get_params" + ] + }, + "MockSplitter": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "MockSplitter" + }, + "__repr__" + ] + } + } + }, + "test_validation": { + "functions": [ + "test_cross_validate", + "check_cross_validate_single_metric", + "check_cross_validate_multi_metric", + "test_cross_val_score_mask", + "test_cross_val_score_score_func", + "test_cross_val_score_errors", + "test_cross_val_score_with_score_func_classification", + "test_cross_val_score_with_score_func_regression", + "test_permutation_score", + "test_permutation_test_score_allow_nans", + "test_cross_val_score_allow_nans", + "test_cross_val_score_multilabel", + "test_learning_curve", + "test_learning_curve_unsupervised", + "test_learning_curve_verbose", + "test_learning_curve_incremental_learning_not_possible", + "test_learning_curve_incremental_learning", + "test_learning_curve_incremental_learning_unsupervised", + "test_learning_curve_n_sample_range_out_of_bounds", + "test_learning_curve_remove_duplicate_sample_sizes", + "test_learning_curve_with_boolean_indices", + "test_validation_curve", + "test_validation_curve_clone_estimator", + "test_validation_curve_cv_splits_consistency", + "test_check_is_permutation", + "test_cross_val_predict_sparse_prediction", + "check_cross_val_predict_binary", + "check_cross_val_predict_multiclass", + "check_cross_val_predict_multilabel", + "check_cross_val_predict_with_method_binary", + "check_cross_val_predict_with_method_multiclass", + "test_cross_val_predict_with_method_multilabel_ovr", + "test_cross_val_predict_with_method_multilabel_rf", + "test_cross_val_predict_with_method_rare_class", + "test_cross_val_predict_with_method_multilabel_rf_rare_class", + "get_expected_predictions", + "test_fit_and_score_failing", + "test_fit_and_score_working", + "three_params_scorer", + "test_score" + ], + "types": { + "MockImprovingEstimator": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "MockImprovingEstimator" + }, + { + "name": "fit", + "updates": [0] + }, + "predict", + "score", + "_is_training_data" + ] + }, + "MockIncrementalImprovingEstimator": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "MockIncrementalImprovingEstimator" + }, + "_is_training_data", + { + "name": "partial_fit", + "updates": [0] + }, + { + "name": "fit", + "updates": [0] + }, + "predict", + "score" + ] + }, + "MockEstimatorWithParameter": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "MockEstimatorWithParameter" + }, + { + "name": "fit", + "updates": [0] + }, + "predict", + "score", + "_is_training_data" + ] + }, + "MockEstimatorWithSingleFitCallAllowed": { + "methods": [ + { + "name": "fit", + "updates": [0] + }, + "predict", + "score" + ] + }, + "MockClassifier": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "MockClassifier" + }, + { + "name": "fit", + "updates": [0] + }, + "predict", + "score", + "get_params" + ] + }, + "BrokenEstimator": { + "methods": [] + }, + "BadCV": { + "methods": ["split"] + }, + "RFWithDecisionFunction": { + "methods": ["decision_function"] + } + } + } + }, + "types": {}, + "functions": [] + } + }, + "types": {}, + "functions": [] + }, + "neighbors": { + "modules": { + "tests": { + "modules": { + "test_ball_tree": { + "functions": [ + "dist_func", + "brute_force_neighbors", + "test_ball_tree_query_radius", + "test_ball_tree_query_radius_distance", + "compute_kernel_slow", + "test_gaussian_kde", + "test_ball_tree_two_point", + "test_ball_tree_pickle", + "test_neighbors_heap", + "test_node_heap", + "test_simultaneous_sort", + "test_query_haversine" + ], + "types": {} + }, + "test_dist_metrics": { + "functions": [ + "dist_func", + "check_cdist", + "check_cdist_bool", + "check_pdist", + { + "name": "check_pdist_bool", + "updates": [1] + }, + "check_pickle", + "test_haversine_metric", + "test_pyfunc_metric", + "test_bad_pyfunc_metric", + "test_input_data_size" + ], + "types": {} + }, + "test_kde": { + "functions": [ + "compute_kernel_slow", + "check_results", + "test_kernel_density_sampling", + "test_kde_score", + "test_kde_badargs", + "test_kde_pipeline_gridsearch", + "test_kde_sample_weights" + ], + "types": {} + }, + "test_kd_tree": { + "functions": [ + "brute_force_neighbors", + "check_neighbors", + "test_kd_tree_query_radius", + "test_kd_tree_query_radius_distance", + "compute_kernel_slow", + "check_results", + "test_gaussian_kde", + "test_neighbors_heap", + "test_node_heap", + "test_simultaneous_sort" + ], + "types": {} + }, + "test_lof": { + "functions": [ + "test_contamination", + "test_contamination_future_warning", + "test_predicted_outlier_number" + ], + "types": {} + }, + "test_nca": { + "functions": [ + "test_simple_example", + "test_toy_example_collapse_points", + "test_finite_differences", + "test_params_validation", + "test_transformation_dimensions", + "test_n_components", + "test_init_transformation", + "test_warm_start_validation", + "test_warm_start_effectiveness", + "test_no_verbose", + "test_singleton_class", + "test_one_class", + "test_callback", + "test_expected_transformation_shape", + "test_convergence_warning" + ], + "types": { + "LossStorer": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "LossStorer" + }, + { + "name": "callback", + "updates": [0] + } + ] + }, + "TransformationStorer": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "TransformationStorer" + }, + { + "name": "callback", + "updates": [0] + } + ] + } + } + }, + "test_nearest_centroid": { + "functions": [ + "test_classification_toy", + "test_precomputed", + "test_iris", + "test_iris_shrinkage", + "test_pickle", + "test_shrinkage_correct", + "test_shrinkage_threshold_decoded_y", + "test_predict_translated_data", + "test_manhattan_metric" + ], + "types": {} + }, + "test_neighbors": { + "functions": [ + "_weight_func", + "test_unsupervised_kneighbors", + "test_unsupervised_inputs", + "test_n_neighbors_datatype", + "test_not_fitted_error_gets_raised", + "test_precomputed", + "test_unsupervised_radius_neighbors", + "test_kneighbors_classifier", + "test_kneighbors_classifier_float_labels", + "test_kneighbors_classifier_predict_proba", + "test_radius_neighbors_classifier", + "test_radius_neighbors_classifier_when_no_neighbors", + "test_radius_neighbors_classifier_outlier_labeling", + "test_radius_neighbors_classifier_zero_distance", + "test_neighbors_regressors_zero_distance", + "test_radius_neighbors_boundary_handling", + "test_RadiusNeighborsClassifier_multioutput", + "test_kneighbors_classifier_sparse", + "test_KNeighborsClassifier_multioutput", + "test_kneighbors_regressor", + "test_KNeighborsRegressor_multioutput_uniform_weight", + "test_kneighbors_regressor_multioutput", + "test_radius_neighbors_regressor", + "test_RadiusNeighborsRegressor_multioutput_with_uniform_weight", + "test_RadiusNeighborsRegressor_multioutput", + "test_kneighbors_regressor_sparse", + "test_neighbors_iris", + "test_neighbors_digits", + "test_kneighbors_graph", + "test_kneighbors_graph_sparse", + "test_radius_neighbors_graph", + "test_radius_neighbors_graph_sparse", + "test_neighbors_badargs", + "test_neighbors_metrics", + "test_callable_metric", + "test_valid_brute_metric_for_auto_algorithm", + "test_metric_params_interface", + "test_predict_sparse_ball_kd_tree", + "test_non_euclidean_kneighbors", + "check_object_arrays", + "test_k_and_radius_neighbors_train_is_not_query", + "test_k_and_radius_neighbors_X_None", + "test_k_and_radius_neighbors_duplicates", + "test_include_self_neighbors_graph", + "test_dtype_convert", + "test_sparse_metric_callable" + ], + "types": {} + }, + "test_quad_tree": { + "functions": [ + "test_quadtree_boundary_computation", + "test_quadtree_similar_point", + "test_summarize" + ], + "types": {} + } + }, + "types": {}, + "functions": [] + }, + "base": { + "functions": [ + "_check_weights", + { + "name": "_get_weights", + "updates": [0] + }, + "_tree_query_parallel_helper", + "_tree_query_radius_parallel_helper" + ], + "types": { + "NeighborsBase": { + "methods": [ + "_check_algorithm_metric", + { + "name": "_fit", + "updates": [0] + } + ] + }, + "KNeighborsMixin": { + "methods": [ + "_kneighbors_reduce_func", + "kneighbors", + "kneighbors_graph" + ] + }, + "RadiusNeighborsMixin": { + "methods": [ + "_radius_neighbors_reduce_func", + "radius_neighbors", + "radius_neighbors_graph" + ] + }, + "SupervisedFloatMixin": { + "methods": [ + { + "name": "fit", + "updates": [0] + } + ] + }, + "SupervisedIntegerMixin": { + "methods": [ + { + "name": "fit", + "updates": [0] + } + ] + }, + "UnsupervisedMixin": { + "methods": ["fit"] + } + } + }, + "classification": { + "functions": [], + "types": { + "KNeighborsClassifier": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "KNeighborsClassifier" + }, + "predict", + "predict_proba" + ] + }, + "RadiusNeighborsClassifier": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "RadiusNeighborsClassifier" + }, + "predict" + ] + } + } + }, + "graph": { + "functions": [ + "_check_params", + "_query_include_self", + "kneighbors_graph", + "radius_neighbors_graph" + ], + "types": {} + }, + "kde": { + "functions": [], + "types": { + "KernelDensity": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "KernelDensity" + }, + "_choose_algorithm", + { + "name": "fit", + "updates": [0] + }, + "score_samples", + "score", + "sample" + ] + } + } + }, + "lof": { + "functions": [], + "types": { + "LocalOutlierFactor": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "LocalOutlierFactor" + }, + { + "name": "_fit_predict", + "updates": [0] + }, + { + "name": "fit", + "updates": [0] + }, + "_predict", + "_decision_function", + "_score_samples", + "_local_reachability_density" + ] + } + } + }, + "nca": { + "functions": [], + "types": { + "NeighborhoodComponentsAnalysis": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "NeighborhoodComponentsAnalysis" + }, + { + "name": "fit", + "updates": [0] + }, + "transform", + "_validate_params", + "_initialize", + { + "name": "_callback", + "updates": [0] + }, + { + "name": "_loss_grad_lbfgs", + "updates": [0] + } + ] + } + } + }, + "nearest_centroid": { + "functions": [], + "types": { + "NearestCentroid": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "NearestCentroid" + }, + { + "name": "fit", + "updates": [0] + }, + "predict" + ] + } + } + }, + "regression": { + "functions": [], + "types": { + "KNeighborsRegressor": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "KNeighborsRegressor" + }, + "predict" + ] + }, + "RadiusNeighborsRegressor": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "RadiusNeighborsRegressor" + }, + "predict" + ] + } + } + }, + "setup": { + "functions": ["configuration"], + "types": {} + }, + "unsupervised": { + "functions": [], + "types": { + "NearestNeighbors": { + "methods": [ + { + "name": "__init__", + "updates": [], + "returns": "NearestNeighbors" + }, + "__init__" + ] + } + } + } + }, + "types": { + "NearestNeighbors": { + "methods": [ + { + "name": "__init__", + "updates": [], + "returns": "NearestNeighbors" + }, + "__init__" + ] + }, + "KNeighborsClassifier": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "KNeighborsClassifier" + }, + "predict", + "predict_proba" + ] + }, + "RadiusNeighborsClassifier": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "RadiusNeighborsClassifier" + }, + "predict" + ] + }, + "KNeighborsRegressor": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "KNeighborsRegressor" + }, + "predict" + ] + }, + "RadiusNeighborsRegressor": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "RadiusNeighborsRegressor" + }, + "predict" + ] + }, + "NearestCentroid": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "NearestCentroid" + }, + { + "name": "fit", + "updates": [0] + }, + "predict" + ] + }, + "KernelDensity": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "KernelDensity" + }, + "_choose_algorithm", + { + "name": "fit", + "updates": [0] + }, + "score_samples", + "score", + "sample" + ] + }, + "LocalOutlierFactor": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "LocalOutlierFactor" + }, + { + "name": "_fit_predict", + "updates": [0] + }, + { + "name": "fit", + "updates": [0] + }, + "_predict", + "_decision_function", + "_score_samples", + "_local_reachability_density" + ] + }, + "NeighborhoodComponentsAnalysis": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "NeighborhoodComponentsAnalysis" + }, + { + "name": "fit", + "updates": [0] + }, + "transform", + "_validate_params", + "_initialize", + { + "name": "_callback", + "updates": [0] + }, + { + "name": "_loss_grad_lbfgs", + "updates": [0] + } + ] + } + }, + "functions": ["kneighbors_graph", "radius_neighbors_graph"] + }, + "neural_network": { + "modules": { + "tests": { + "modules": { + "test_mlp": { + "functions": [ + "test_alpha", + "test_fit", + "test_gradient", + "test_lbfgs_classification", + "test_lbfgs_regression", + "test_learning_rate_warmstart", + "test_multilabel_classification", + "test_partial_fit_classes_error", + "test_partial_fit_classification", + "test_partial_fit_unseen_classes", + "test_partial_fit_regression", + "test_partial_fit_errors", + "test_params_errors", + "test_predict_proba_binary", + "test_predict_proba_multiclass", + "test_predict_proba_multilabel", + "test_shuffle", + "test_sparse_matrices", + "test_tolerance", + "test_verbose_sgd", + "test_early_stopping", + "test_adaptive_learning_rate", + "test_n_iter_no_change", + "test_early_stopping_stratified" + ], + "types": {} + }, + "test_rbm": { + "functions": [ + "test_fit", + "test_partial_fit", + "test_transform", + "test_small_sparse", + "test_small_sparse_partial_fit", + "test_sample_hiddens", + "test_fit_gibbs", + "test_fit_gibbs_sparse", + "test_gibbs_smoke", + "test_score_samples", + "test_rbm_verbose", + "test_sparse_and_verbose" + ], + "types": {} + }, + "test_stochastic_optimizers": { + "functions": [ + "test_base_optimizer", + "test_sgd_optimizer_no_momentum", + "test_sgd_optimizer_momentum", + "test_sgd_optimizer_trigger_stopping", + "test_sgd_optimizer_nesterovs_momentum", + "test_adam_optimizer" + ], + "types": {} + } + }, + "types": {}, + "functions": [] + }, + "multilayer_perceptron": { + "functions": ["_pack"], + "types": { + "BaseMultilayerPerceptron": { + "methods": [ + "_unpack", + { + "name": "_forward_pass", + "updates": [1] + }, + { + "name": "_compute_loss_grad", + "updates": [5, 6] + }, + { + "name": "_loss_grad_lbfgs", + "updates": [0, 4] + }, + { + "name": "_backprop", + "updates": [4, 5, 6] + }, + { + "name": "_initialize", + "updates": [0] + }, + "_init_coef", + { + "name": "_fit", + "updates": [0, 3] + }, + "_validate_hyperparameters", + { + "name": "_fit_lbfgs", + "updates": [0] + }, + { + "name": "_fit_stochastic", + "updates": [0, 3, 4, 5, 6] + }, + { + "name": "_update_no_improvement_count", + "updates": [0] + }, + { + "name": "fit", + "updates": [0] + }, + { + "name": "_partial_fit", + "updates": [0] + }, + "_predict" + ] + }, + "MLPClassifier": { + "methods": [ + { + "name": "__init__", + "updates": [], + "returns": "MLPClassifier" + }, + "__init__", + { + "name": "_validate_input", + "updates": [0] + }, + "predict", + "fit", + { + "name": "_partial_fit", + "updates": [0] + }, + "predict_log_proba", + "predict_proba" + ] + }, + "MLPRegressor": { + "methods": [ + { + "name": "__init__", + "updates": [], + "returns": "MLPRegressor" + }, + "__init__", + "predict", + "_validate_input", + { + "name": "fit", + "updates": [0] + } + ] + } + } + }, + "rbm": { + "functions": [], + "types": { + "BernoulliRBM": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "BernoulliRBM" + }, + "transform", + "_mean_hiddens", + "_sample_hiddens", + "_sample_visibles", + "_free_energy", + { + "name": "gibbs", + "updates": [0] + }, + { + "name": "partial_fit", + "updates": [0] + }, + { + "name": "_fit", + "updates": [0] + }, + "score_samples", + { + "name": "fit", + "updates": [0] + } + ] + } + } + } + }, + "types": { + "BernoulliRBM": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "BernoulliRBM" + }, + "transform", + "_mean_hiddens", + "_sample_hiddens", + "_sample_visibles", + "_free_energy", + { + "name": "gibbs", + "updates": [0] + }, + { + "name": "partial_fit", + "updates": [0] + }, + { + "name": "_fit", + "updates": [0] + }, + "score_samples", + { + "name": "fit", + "updates": [0] + } + ] + }, + "MLPClassifier": { + "methods": [ + { + "name": "__init__", + "updates": [], + "returns": "MLPClassifier" + }, + "__init__", + { + "name": "_validate_input", + "updates": [0] + }, + "predict", + "fit", + { + "name": "_partial_fit", + "updates": [0] + }, + "predict_log_proba", + "predict_proba" + ] + }, + "MLPRegressor": { + "methods": [ + { + "name": "__init__", + "updates": [], + "returns": "MLPRegressor" + }, + "__init__", + "predict", + "_validate_input", + { + "name": "fit", + "updates": [0] + } + ] + } + }, + "functions": [] + }, + "preprocessing": { + "modules": { + "tests": { + "modules": { + "test_base": { + "functions": [ + "toarray", + "_check_transform_selected", + "test_transform_selected_retain_order" + ], + "types": {} + }, + "test_common": { + "functions": ["_get_valid_samples_by_column"], + "types": {} + }, + "test_data": { + "functions": [ + "toarray", + "_check_dim_1axis", + "assert_correct_incr", + "test_polynomial_features", + "test_polynomial_feature_names", + "test_polynomial_feature_array_order", + "test_standard_scaler_1d", + "test_standard_scaler_dtype", + "test_scale_1d", + "test_scaler_2d_arrays", + "test_scaler_float16_overflow", + "test_handle_zeros_in_scale", + "test_minmax_scaler_partial_fit", + "test_standard_scaler_partial_fit", + "test_standard_scaler_partial_fit_numerical_stability", + "test_partial_fit_sparse_input", + "test_standard_scaler_trasform_with_partial_fit", + "test_min_max_scaler_iris", + "test_min_max_scaler_zero_variance_features", + "test_minmax_scale_axis1", + "test_min_max_scaler_1d", + "test_scaler_without_centering", + "_check_identity_scalers_attributes", + "test_scaler_return_identity", + "test_scaler_int", + "test_scaler_without_copy", + "test_scale_sparse_with_mean_raise_exception", + "test_scale_input_finiteness_validation", + "test_robust_scaler_error_sparse", + "test_robust_scaler_col_zero_sparse", + "test_robust_scaler_2d_arrays", + "test_robust_scaler_transform_one_row_csr", + "test_robust_scaler_iris", + "test_robust_scaler_iris_quantiles", + "test_quantile_transform_iris", + "test_quantile_transform_check_error", + "test_quantile_transform_sparse_ignore_zeros", + "test_quantile_transform_dense_toy", + "test_quantile_transform_subsampling", + "test_quantile_transform_sparse_toy", + "test_quantile_transform_bounds", + "test_quantile_transform_and_inverse", + "test_quantile_transform_nan", + "test_deprecated_quantile_transform_copy", + "test_robust_scaler_invalid_range", + "test_scale_function_without_centering", + "test_robust_scale_axis1", + "test_robust_scale_1d_array", + "test_robust_scaler_zero_variance_features", + "test_maxabs_scaler_zero_variance_features", + "test_maxabs_scaler_large_negative_value", + "test_maxabs_scaler_transform_one_row_csr", + "test_maxabs_scaler_1d", + "test_maxabs_scaler_partial_fit", + "test_normalizer_l1", + "test_normalizer_l2", + "test_normalizer_max", + "test_normalize", + "test_binarizer", + "test_center_kernel", + "test_cv_pipeline_precomputed", + "test_fit_transform", + "test_add_dummy_feature", + "test_add_dummy_feature_coo", + "test_add_dummy_feature_csc", + "test_add_dummy_feature_csr", + "test_fit_cold_start", + "test_power_transformer_1d", + "test_power_transformer_2d", + "test_power_transformer_boxcox_strictly_positive_exception", + "test_power_transformer_method_exception", + "test_power_transformer_lambda_zero", + "test_power_transformer_lambda_one", + "test_yeo_johnson_darwin_example", + "test_power_transform_default_method" + ], + "types": {} + }, + "test_discretization": { + "functions": [ + "test_valid_n_bins", + "test_invalid_n_bins", + "test_invalid_n_bins_array", + "test_invalid_n_features", + "test_transform_1d_behavior", + "test_numeric_stability", + "test_invalid_encode_option", + "test_encode_options", + "test_invalid_strategy_option", + "test_overwrite", + "test_percentile_numeric_stability" + ], + "types": {} + }, + "test_encoders": { + "functions": [ + "toarray", + "test_one_hot_encoder_sparse", + "test_one_hot_encoder_dense", + "test_one_hot_encoder_deprecationwarnings", + "test_one_hot_encoder_force_new_behaviour", + "_run_one_hot", + "_check_one_hot", + "test_one_hot_encoder_categorical_features", + "test_one_hot_encoder_categorical_features_ignore_unknown", + "test_one_hot_encoder_handle_unknown", + "test_one_hot_encoder_not_fitted", + "test_one_hot_encoder_no_categorical_features", + "test_one_hot_encoder_handle_unknown_strings", + "test_one_hot_encoder_set_params", + "check_categorical_onehot", + "test_one_hot_encoder_unsorted_categories", + "test_one_hot_encoder_specified_categories_mixed_columns", + "test_one_hot_encoder_pandas", + "test_one_hot_encoder_feature_names", + "test_one_hot_encoder_feature_names_unicode", + "test_ordinal_encoder_inverse", + "test_ordinal_encoder_raise_categories_shape", + "test_encoder_dtypes", + "test_encoder_dtypes_pandas", + "test_one_hot_encoder_warning", + "test_one_hot_encoder_drop_manual", + "test_one_hot_encoder_invalid_params" + ], + "types": {} + }, + "test_function_transformer": { + "functions": [ + "_make_func", + "test_delegate_to_func", + "test_function_transformer_frame" + ], + "types": {} + }, + "test_imputation": { + "functions": ["safe_median", "safe_mean"], + "types": {} + }, + "test_label": { + "functions": [ + "toarray", + "test_label_binarizer", + "test_label_binarizer_unseen_labels", + "test_label_binarizer_set_label_encoding", + "test_label_encoder_negative_ints", + "test_label_encoder_errors", + "test_sparse_output_multilabel_binarizer", + "test_multilabel_binarizer", + "test_multilabel_binarizer_empty_sample", + "test_multilabel_binarizer_unknown_class", + "test_multilabel_binarizer_given_classes", + "test_multilabel_binarizer_multiple_calls", + "test_multilabel_binarizer_same_length_sequence", + "test_multilabel_binarizer_non_integer_labels", + "test_multilabel_binarizer_non_unique", + "test_multilabel_binarizer_inverse_validation", + "test_label_binarize_with_class_order", + "check_binarized_results", + "test_label_binarize_binary", + "test_label_binarize_multiclass", + "test_label_binarize_multilabel", + "test_invalid_input_label_binarize", + "test_inverse_binarize_multiclass" + ], + "types": {} + } + }, + "types": {}, + "functions": [] + }, + "base": { + "functions": [ + { + "name": "_transform_selected", + "updates": [0] + } + ], + "types": {} + }, + "data": { + "functions": [ + { + "name": "_handle_zeros_in_scale", + "updates": [0] + }, + "scale", + "minmax_scale", + "maxabs_scale", + "robust_scale", + "normalize", + { + "name": "binarize", + "updates": [0] + }, + "add_dummy_feature", + "quantile_transform", + "power_transform" + ], + "types": { + "MinMaxScaler": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "MinMaxScaler" + }, + "_reset", + { + "name": "fit", + "updates": [0] + }, + { + "name": "partial_fit", + "updates": [0] + }, + "transform", + "inverse_transform", + "_more_tags" + ] + }, + "StandardScaler": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "StandardScaler" + }, + "_reset", + { + "name": "fit", + "updates": [0] + }, + { + "name": "partial_fit", + "updates": [0] + }, + "transform", + "inverse_transform", + "_more_tags" + ] + }, + "MaxAbsScaler": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "MaxAbsScaler" + }, + "_reset", + { + "name": "fit", + "updates": [0] + }, + { + "name": "partial_fit", + "updates": [0] + }, + "transform", + "inverse_transform", + "_more_tags" + ] + }, + "RobustScaler": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "RobustScaler" + }, + { + "name": "fit", + "updates": [0] + }, + "transform", + "inverse_transform", + "_more_tags" + ] + }, + "PolynomialFeatures": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "PolynomialFeatures" + }, + "get_feature_names", + { + "name": "fit", + "updates": [0] + }, + "transform" + ] + }, + "Normalizer": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "Normalizer" + }, + "fit", + "transform", + "_more_tags" + ] + }, + "Binarizer": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "Binarizer" + }, + "fit", + "transform", + "_more_tags" + ] + }, + "KernelCenterer": { + "methods": [ + { + "name": "__init__", + "updates": [], + "returns": "KernelCenterer" + }, + "__init__", + { + "name": "fit", + "updates": [0] + }, + "transform" + ] + }, + "QuantileTransformer": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "QuantileTransformer" + }, + { + "name": "_dense_fit", + "updates": [0] + }, + { + "name": "_sparse_fit", + "updates": [0] + }, + { + "name": "fit", + "updates": [0] + }, + { + "name": "_transform_col", + "updates": [1] + }, + "_check_inputs", + "_check_is_fitted", + { + "name": "_transform", + "updates": [1] + }, + { + "name": "transform", + "updates": [1] + }, + { + "name": "inverse_transform", + "updates": [1] + }, + "_more_tags" + ] + }, + "PowerTransformer": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "PowerTransformer" + }, + { + "name": "fit", + "updates": [0, 1] + }, + { + "name": "fit_transform", + "updates": [0, 1] + }, + { + "name": "_fit", + "updates": [0, 1] + }, + { + "name": "transform", + "updates": [1] + }, + { + "name": "inverse_transform", + "updates": [1] + }, + "_box_cox_inverse_tranform", + "_yeo_johnson_inverse_transform", + "_yeo_johnson_transform", + "_box_cox_optimize", + "_yeo_johnson_optimize", + "_check_input", + "_more_tags" + ] + } + } + }, + "imputation": { + "functions": ["_get_mask", "_most_frequent"], + "types": { + "Imputer": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "Imputer" + }, + { + "name": "fit", + "updates": [0] + }, + "_sparse_fit", + "_dense_fit", + { + "name": "transform", + "updates": [1] + }, + "_more_tags" + ] + } + } + }, + "label": { + "functions": [ + "_encode_numpy", + "_encode_python", + "_encode", + "_encode_check_unknown", + "label_binarize", + "_inverse_binarize_multiclass", + { + "name": "_inverse_binarize_thresholding", + "updates": [0] + } + ], + "types": { + "LabelEncoder": { + "methods": [ + { + "name": "fit", + "updates": [0] + }, + { + "name": "fit_transform", + "updates": [0] + }, + "transform", + "inverse_transform", + "_more_tags" + ] + }, + "LabelBinarizer": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "LabelBinarizer" + }, + { + "name": "fit", + "updates": [0] + }, + { + "name": "fit_transform", + "updates": [0] + }, + "transform", + "inverse_transform", + "_more_tags" + ] + }, + "MultiLabelBinarizer": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "MultiLabelBinarizer" + }, + { + "name": "fit", + "updates": [0] + }, + { + "name": "fit_transform", + "updates": [0] + }, + { + "name": "transform", + "updates": [0] + }, + { + "name": "_build_cache", + "updates": [0] + }, + "_transform", + "inverse_transform", + "_more_tags" + ] + } + } + }, + "setup": { + "functions": ["configuration"], + "types": {} + } + }, + "types": { + "Binarizer": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "Binarizer" + }, + "fit", + "transform", + "_more_tags" + ] + }, + "KernelCenterer": { + "methods": [ + { + "name": "__init__", + "updates": [], + "returns": "KernelCenterer" + }, + "__init__", + { + "name": "fit", + "updates": [0] + }, + "transform" + ] + }, + "MinMaxScaler": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "MinMaxScaler" + }, + "_reset", + { + "name": "fit", + "updates": [0] + }, + { + "name": "partial_fit", + "updates": [0] + }, + "transform", + "inverse_transform", + "_more_tags" + ] + }, + "MaxAbsScaler": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "MaxAbsScaler" + }, + "_reset", + { + "name": "fit", + "updates": [0] + }, + { + "name": "partial_fit", + "updates": [0] + }, + "transform", + "inverse_transform", + "_more_tags" + ] + }, + "Normalizer": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "Normalizer" + }, + "fit", + "transform", + "_more_tags" + ] + }, + "RobustScaler": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "RobustScaler" + }, + { + "name": "fit", + "updates": [0] + }, + "transform", + "inverse_transform", + "_more_tags" + ] + }, + "StandardScaler": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "StandardScaler" + }, + "_reset", + { + "name": "fit", + "updates": [0] + }, + { + "name": "partial_fit", + "updates": [0] + }, + "transform", + "inverse_transform", + "_more_tags" + ] + }, + "QuantileTransformer": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "QuantileTransformer" + }, + { + "name": "_dense_fit", + "updates": [0] + }, + { + "name": "_sparse_fit", + "updates": [0] + }, + { + "name": "fit", + "updates": [0] + }, + { + "name": "_transform_col", + "updates": [1] + }, + "_check_inputs", + "_check_is_fitted", + { + "name": "_transform", + "updates": [1] + }, + { + "name": "transform", + "updates": [1] + }, + { + "name": "inverse_transform", + "updates": [1] + }, + "_more_tags" + ] + }, + "PowerTransformer": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "PowerTransformer" + }, + { + "name": "fit", + "updates": [0, 1] + }, + { + "name": "fit_transform", + "updates": [0, 1] + }, + { + "name": "_fit", + "updates": [0, 1] + }, + { + "name": "transform", + "updates": [1] + }, + { + "name": "inverse_transform", + "updates": [1] + }, + "_box_cox_inverse_tranform", + "_yeo_johnson_inverse_transform", + "_yeo_johnson_transform", + "_box_cox_optimize", + "_yeo_johnson_optimize", + "_check_input", + "_more_tags" + ] + }, + "PolynomialFeatures": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "PolynomialFeatures" + }, + "get_feature_names", + { + "name": "fit", + "updates": [0] + }, + "transform" + ] + }, + "LabelBinarizer": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "LabelBinarizer" + }, + { + "name": "fit", + "updates": [0] + }, + { + "name": "fit_transform", + "updates": [0] + }, + "transform", + "inverse_transform", + "_more_tags" + ] + }, + "LabelEncoder": { + "methods": [ + { + "name": "fit", + "updates": [0] + }, + { + "name": "fit_transform", + "updates": [0] + }, + "transform", + "inverse_transform", + "_more_tags" + ] + }, + "MultiLabelBinarizer": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "MultiLabelBinarizer" + }, + { + "name": "fit", + "updates": [0] + }, + { + "name": "fit_transform", + "updates": [0] + }, + { + "name": "transform", + "updates": [0] + }, + { + "name": "_build_cache", + "updates": [0] + }, + "_transform", + "inverse_transform", + "_more_tags" + ] + }, + "Imputer": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "Imputer" + }, + { + "name": "fit", + "updates": [0] + }, + "_sparse_fit", + "_dense_fit", + { + "name": "transform", + "updates": [1] + }, + "_more_tags" + ] + } + }, + "functions": [ + "add_dummy_feature", + { + "name": "binarize", + "updates": [0] + }, + "normalize", + "scale", + "robust_scale", + "maxabs_scale", + "minmax_scale", + "quantile_transform", + "power_transform", + "label_binarize" + ] + }, + "semi_supervised": { + "modules": { + "tests": { + "modules": { + "test_label_propagation": { + "functions": [ + "test_fit_transduction", + "test_distribution", + "test_predict", + "test_predict_proba", + "test_label_spreading_closed_form", + "test_label_propagation_closed_form", + "test_valid_alpha", + "test_convergence_speed", + "test_convergence_warning" + ], + "types": {} + } + }, + "types": {}, + "functions": [] + }, + "label_propagation": { + "functions": [], + "types": { + "BaseLabelPropagation": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "BaseLabelPropagation" + }, + { + "name": "_get_kernel", + "updates": [0] + }, + "predict", + { + "name": "predict_proba", + "updates": [0] + }, + { + "name": "fit", + "updates": [0] + } + ] + }, + "LabelPropagation": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "LabelPropagation" + }, + "__init__", + { + "name": "_build_graph", + "updates": [0] + }, + { + "name": "fit", + "updates": [0] + }, + "predict", + { + "name": "predict_proba", + "updates": [0] + } + ] + }, + "LabelSpreading": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "LabelSpreading" + }, + "__init__", + { + "name": "_build_graph", + "updates": [0] + }, + "predict", + { + "name": "predict_proba", + "updates": [0] + }, + { + "name": "fit", + "updates": [0] + } + ] + } + } + } + }, + "types": { + "LabelPropagation": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "LabelPropagation" + }, + "__init__", + { + "name": "_build_graph", + "updates": [0] + }, + { + "name": "fit", + "updates": [0] + }, + "predict", + { + "name": "predict_proba", + "updates": [0] + } + ] + }, + "LabelSpreading": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "LabelSpreading" + }, + "__init__", + { + "name": "_build_graph", + "updates": [0] + }, + "predict", + { + "name": "predict_proba", + "updates": [0] + }, + { + "name": "fit", + "updates": [0] + } + ] + } + }, + "functions": [] + }, + "svm": { + "modules": { + "tests": { + "modules": { + "test_bounds": { + "functions": [ + "test_l1_min_c_l2_loss", + "check_l1_min_c", + "test_ill_posed_min_c", + "test_unsupported_loss" + ], + "types": {} + }, + "test_sparse": { + "functions": [ + "check_svm_model_equal", + "test_unsorted_indices", + "test_svc_with_custom_kernel", + "test_svc_iris", + "test_sparse_decision_function", + "test_error", + "test_linearsvc", + "test_linearsvc_iris", + "test_sample_weights", + "test_sparse_liblinear_intercept_handling", + "test_sparse_realdata", + "test_sparse_svc_clone_with_callable_kernel", + "test_timeout", + "test_consistent_proba" + ], + "types": {} + }, + "test_svm": { + "functions": [ + "test_libsvm_parameters", + "test_libsvm_iris", + "test_precomputed", + "test_svr", + "test_linearsvr", + "test_linearsvr_fit_sampleweight", + "test_svr_errors", + "test_oneclass", + "test_oneclass_decision_function", + "test_oneclass_score_samples", + "test_tweak_params", + "test_probability", + "test_decision_function", + "test_decision_function_shape", + "test_svr_predict", + "test_sample_weights", + "test_bad_input", + "test_unicode_kernel", + "test_sparse_precomputed", + "test_linearsvc_parameters", + "test_linearsvx_loss_penalty_deprecations", + "test_linear_svx_uppercase_loss_penality_raises_error", + "test_linearsvc", + "test_linearsvc_crammer_singer", + "test_linearsvc_fit_sampleweight", + "test_crammer_singer_binary", + "test_linearsvc_iris", + "test_dense_liblinear_intercept_handling", + "test_liblinear_set_coef", + "test_immutable_coef_property", + "test_linearsvc_verbose", + "test_svc_clone_with_callable_kernel", + "test_svc_bad_kernel", + "test_timeout", + "test_unfitted", + "test_linear_svm_convergence_warnings", + "test_svr_coef_sign", + "test_linear_svc_intercept_scaling", + "test_lsvc_intercept_scaling_zero", + "test_hasattr_predict_proba", + "test_decision_function_shape_two_class", + "test_ovr_decision_function", + "test_gamma_auto", + "test_gamma_scale" + ], + "types": {} + } + }, + "types": {}, + "functions": [] + }, + "base": { + "functions": [ + "_one_vs_one_coef", + "_get_liblinear_solver_type", + "_fit_liblinear" + ], + "types": { + "BaseLibSVM": { + "methods": [ + { + "name": "fit", + "updates": [0] + }, + { + "name": "_validate_targets", + "updates": [0] + }, + "_warn_from_fit_status", + { + "name": "_dense_fit", + "updates": [0] + }, + { + "name": "_sparse_fit", + "updates": [1, 0] + }, + "predict", + "_dense_predict", + "_sparse_predict", + "_compute_kernel", + { + "name": "_decision_function", + "updates": [1] + }, + "_dense_decision_function", + { + "name": "_sparse_decision_function", + "updates": [1] + }, + "_validate_for_predict", + "_get_coef" + ] + }, + "BaseSVC": { + "methods": [ + { + "name": "_validate_targets", + "updates": [0] + }, + "decision_function", + "predict", + "_check_proba", + "_predict_proba", + "_predict_log_proba", + "_dense_predict_proba", + { + "name": "_sparse_predict_proba", + "updates": [1] + }, + "_get_coef", + { + "name": "fit", + "updates": [0] + } + ] + } + } + }, + "bounds": { + "functions": ["l1_min_c"], + "types": {} + }, + "classes": { + "functions": [], + "types": { + "LinearSVC": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "LinearSVC" + }, + { + "name": "fit", + "updates": [0] + }, + "predict", + "score" + ] + }, + "LinearSVR": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "LinearSVR" + }, + { + "name": "fit", + "updates": [0] + }, + "predict", + "score" + ] + }, + "SVC": { + "methods": [ + { + "name": "__init__", + "updates": [], + "returns": "SVC" + }, + "__init__" + ] + }, + "NuSVC": { + "methods": [ + { + "name": "__init__", + "updates": [], + "returns": "NuSVC" + }, + "__init__" + ] + }, + "SVR": { + "methods": [ + { + "name": "__init__", + "updates": [], + "returns": "SVR" + }, + "__init__" + ] + }, + "NuSVR": { + "methods": [ + { + "name": "__init__", + "updates": [], + "returns": "NuSVR" + }, + "__init__" + ] + }, + "OneClassSVM": { + "methods": [ + { + "name": "__init__", + "updates": [], + "returns": "OneClassSVM" + }, + "__init__", + { + "name": "fit", + "updates": [0] + }, + "decision_function", + "score_samples", + "predict" + ] + } + } + }, + "setup": { + "functions": ["configuration"], + "types": {} + } + }, + "types": { + "SVC": { + "methods": [ + { + "name": "__init__", + "updates": [], + "returns": "SVC" + }, + "__init__" + ] + }, + "NuSVC": { + "methods": [ + { + "name": "__init__", + "updates": [], + "returns": "NuSVC" + }, + "__init__" + ] + }, + "SVR": { + "methods": [ + { + "name": "__init__", + "updates": [], + "returns": "SVR" + }, + "__init__" + ] + }, + "NuSVR": { + "methods": [ + { + "name": "__init__", + "updates": [], + "returns": "NuSVR" + }, + "__init__" + ] + }, + "OneClassSVM": { + "methods": [ + { + "name": "__init__", + "updates": [], + "returns": "OneClassSVM" + }, + "__init__", + { + "name": "fit", + "updates": [0] + }, + "decision_function", + "score_samples", + "predict" + ] + }, + "LinearSVC": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "LinearSVC" + }, + { + "name": "fit", + "updates": [0] + } + ] + }, + "LinearSVR": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "LinearSVR" + }, + { + "name": "fit", + "updates": [0] + } + ] + } + }, + "functions": ["l1_min_c"] + }, + "tests": { + "modules": { + "test_base": { + "functions": [ + "test_clone", + "test_clone_2", + "test_clone_buggy", + "test_clone_empty_array", + "test_clone_nan", + "test_clone_sparse_matrices", + "test_clone_estimator_types", + "test_repr", + "test_str", + "test_get_params", + "test_is_classifier", + "test_set_params", + "test_set_params_passes_all_parameters", + "test_set_params_updates_valid_params", + "test_score_sample_weight", + "test_clone_pandas_dataframe", + "test_pickle_version_warning_is_not_raised_with_matching_version", + "test_pickle_version_warning_is_issued_upon_different_version", + "test_pickle_version_warning_is_issued_when_no_version_info_in_pickle", + "test_pickle_version_no_warning_is_issued_with_non_sklearn_estimator", + "test_pickling_when_getstate_is_overwritten_by_mixin", + "test_pickling_when_getstate_is_overwritten_by_mixin_outside_of_sklearn", + "test_tag_inheritance", + "test_regressormixin_score_multioutput" + ], + "types": { + "MyEstimator": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "MyEstimator" + } + ] + }, + "K": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "K" + } + ] + }, + "T": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "T" + } + ] + }, + "NaNTag": { + "methods": ["_more_tags"] + }, + "NoNaNTag": { + "methods": ["_more_tags"] + }, + "OverrideTag": { + "methods": ["_more_tags"] + }, + "DiamondOverwriteTag": { + "methods": [] + }, + "ModifyInitParams": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "ModifyInitParams" + } + ] + }, + "Buggy": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "Buggy" + } + ] + }, + "NoEstimator": { + "methods": [ + { + "name": "__init__", + "updates": [], + "returns": "NoEstimator" + }, + "__init__", + "fit", + "predict" + ] + }, + "VargEstimator": { + "methods": [ + { + "name": "__init__", + "updates": [], + "returns": "VargEstimator" + }, + "__init__" + ] + }, + "TestDecisionTree": { + "methods": ["set_params"] + }, + "DummyEstimator": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "DummyEstimator" + }, + "fit", + "transform" + ] + }, + "TreeBadVersion": { + "methods": ["__getstate__"] + }, + "TreeNoVersion": { + "methods": ["__getstate__"] + }, + "DontPickleAttributeMixin": { + "methods": [ + "__getstate__", + { + "name": "__setstate__", + "updates": [1] + } + ] + }, + "MultiInheritanceEstimator": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "MultiInheritanceEstimator" + } + ] + }, + "SingleInheritanceEstimator": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "SingleInheritanceEstimator" + }, + "__getstate__" + ] + } + } + }, + "test_calibration": { + "functions": [ + "test_calibration_multiclass", + "test_calibration_prefit", + "test_sigmoid_calibration", + "test_calibration_curve", + "test_calibration_nan_imputer", + "test_calibration_prob_sum", + "test_calibration_less_classes" + ], + "types": { + "MockTensorClassifier": { + "methods": [ + { + "name": "fit", + "updates": [0] + }, + "decision_function" + ] + } + } + }, + "test_check_build": { + "functions": ["test_raise_build_error"], + "types": {} + }, + "test_common": { + "functions": [ + "test_all_estimator_no_base_class", + "_tested_estimators", + "_generate_checks_per_estimator", + "_rename_partial", + "_tested_linear_classifiers", + "test_root_import_all_completeness", + "test_all_tests_are_importable" + ], + "types": {} + }, + "test_config": { + "functions": [ + "test_config_context", + "test_config_context_exception", + "test_set_config" + ], + "types": {} + }, + "test_discriminant_analysis": { + "functions": [ + "test_lda_predict", + "test_lda_priors", + "test_lda_coefs", + "test_lda_transform", + "test_lda_explained_variance_ratio", + "test_lda_orthogonality", + "test_lda_scaling", + "test_lda_store_covariance", + "test_lda_numeric_consistency_float32_float64", + "test_qda", + "test_qda_priors", + "test_qda_store_covariance", + "test_qda_regularization", + "test_covariance" + ], + "types": {} + }, + "test_docstring_parameters": { + "functions": [], + "types": {} + }, + "test_dummy": { + "functions": [ + "_check_behavior_2d", + "_check_behavior_2d_for_constant", + "_check_equality_regressor", + "test_most_frequent_and_prior_strategy", + "test_most_frequent_and_prior_strategy_with_2d_column_y", + "test_most_frequent_and_prior_strategy_multioutput", + "test_stratified_strategy", + "test_stratified_strategy_multioutput", + "test_uniform_strategy", + "test_uniform_strategy_multioutput", + "test_string_labels", + "test_classifier_exceptions", + "test_mean_strategy_regressor", + "test_mean_strategy_multioutput_regressor", + "test_regressor_exceptions", + "test_median_strategy_regressor", + "test_median_strategy_multioutput_regressor", + "test_quantile_strategy_regressor", + "test_quantile_strategy_multioutput_regressor", + "test_quantile_invalid", + "test_quantile_strategy_empty_train", + "test_constant_strategy_regressor", + "test_constant_strategy_multioutput_regressor", + "test_y_mean_attribute_regressor", + "test_unknown_strategey_regressor", + "test_constants_not_specified_regressor", + "test_constant_size_multioutput_regressor", + "test_constant_strategy", + "test_constant_strategy_multioutput", + "test_constant_strategy_exceptions", + "test_classification_sample_weight", + "test_constant_strategy_sparse_target", + "test_uniform_strategy_sparse_target_warning", + "test_stratified_strategy_sparse_target", + "test_most_frequent_and_prior_strategy_sparse_target", + "test_dummy_regressor_sample_weight", + "test_dummy_regressor_on_3D_array", + "test_dummy_classifier_on_3D_array", + "test_dummy_regressor_return_std" + ], + "types": {} + }, + "test_init": { + "functions": ["test_import_skl"], + "types": {} + }, + "test_isotonic": { + "functions": [ + "test_permutation_invariance", + "test_check_increasing_small_number_of_samples", + "test_check_increasing_up", + "test_check_increasing_up_extreme", + "test_check_increasing_down", + "test_check_increasing_down_extreme", + "test_check_ci_warn", + "test_isotonic_regression", + "test_isotonic_regression_ties_min", + "test_isotonic_regression_ties_max", + "test_isotonic_regression_ties_secondary_", + "test_isotonic_regression_with_ties_in_differently_sized_groups", + "test_isotonic_regression_reversed", + "test_isotonic_regression_auto_decreasing", + "test_isotonic_regression_auto_increasing", + "test_assert_raises_exceptions", + "test_isotonic_sample_weight_parameter_default_value", + "test_isotonic_min_max_boundaries", + "test_isotonic_sample_weight", + "test_isotonic_regression_oob_raise", + "test_isotonic_regression_oob_clip", + "test_isotonic_regression_oob_nan", + "test_isotonic_regression_oob_bad", + "test_isotonic_regression_oob_bad_after", + "test_isotonic_regression_pickle", + "test_isotonic_duplicate_min_entry", + "test_isotonic_ymin_ymax", + "test_isotonic_zero_weight_loop", + "test_fast_predict", + "test_isotonic_copy_before_fit", + "test_isotonic_dtype", + "test_make_unique_dtype" + ], + "types": {} + }, + "test_kernel_approximation": { + "functions": [ + "test_additive_chi2_sampler", + "test_skewed_chi2_sampler", + "test_rbf_sampler", + "test_input_validation", + "test_nystroem_approximation", + "test_nystroem_default_parameters", + "test_nystroem_singular_kernel", + "test_nystroem_poly_kernel_params", + "test_nystroem_callable" + ], + "types": {} + }, + "test_kernel_ridge": { + "functions": [ + "test_kernel_ridge", + "test_kernel_ridge_csr", + "test_kernel_ridge_csc", + "test_kernel_ridge_singular_kernel", + "test_kernel_ridge_precomputed", + "test_kernel_ridge_precomputed_kernel_unchanged", + "test_kernel_ridge_sample_weights", + "test_kernel_ridge_multi_output" + ], + "types": {} + }, + "test_metaestimators": { + "functions": [], + "types": { + "DelegatorData": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "DelegatorData" + } + ] + }, + "SubEstimator": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "SubEstimator" + }, + { + "name": "fit", + "updates": [0] + }, + "_check_fit" + ] + } + } + }, + "test_multiclass": { + "functions": [ + "test_ovr_exceptions", + "test_check_classification_targets", + "test_ovr_fit_predict", + "test_ovr_partial_fit_exceptions", + "test_ovr_ovo_regressor", + "test_ovr_fit_predict_sparse", + "test_ovr_multiclass", + "test_ovr_multilabel", + "test_ovr_fit_predict_svc", + "test_ovr_multilabel_dataset", + "test_ovr_single_label_predict_proba", + "test_ovr_multilabel_decision_function", + "test_ovr_single_label_decision_function", + "test_ovr_pipeline", + "test_ovr_coef_", + "test_ovr_coef_exceptions", + "test_ovo_exceptions", + "test_ovo_fit_on_list", + "test_ovo_fit_predict", + "test_ovo_partial_fit_predict", + "test_ovo_decision_function", + "test_ovo_string_y", + "test_ovo_one_class", + "test_ovo_float_y", + "test_ecoc_exceptions", + "test_ecoc_fit_predict", + "test_ecoc_float_y", + "test_pairwise_indices", + "test_pairwise_attribute" + ], + "types": {} + }, + "test_multioutput": { + "functions": [ + "test_multi_target_regression", + "test_multi_target_regression_one_target", + "test_multi_target_sparse_regression", + "test_multi_target_sample_weights_api", + "test_multi_target_sample_weights", + "test_multi_output_classification_partial_fit_parallelism", + "test_multi_output_predict_proba", + "test_multi_output_classification", + "test_multiclass_multioutput_estimator", + "test_multiclass_multioutput_estimator_predict_proba", + "test_multi_output_classification_sample_weights", + "test_multi_output_exceptions", + "generate_multilabel_dataset_with_correlations", + "test_classifier_chain_fit_and_predict_with_linear_svc" + ], + "types": {} + }, + "test_naive_bayes": { + "functions": [ + "test_gnb", + "test_gnb_prior", + "test_gnb_sample_weight", + "test_gnb_neg_priors", + "test_gnb_priors", + "test_gnb_priors_sum_isclose", + "test_gnb_wrong_nb_priors", + "test_gnb_prior_greater_one", + "test_gnb_prior_large_bias", + "test_check_update_with_no_data", + "test_gnb_pfit_wrong_nb_features", + "test_discrete_prior", + "check_partial_fit", + "test_mnb_prior_unobserved_targets", + "test_gnb_partial_fit", + "test_discretenb_predict_proba", + "check_sample_weight_multiclass", + "test_sample_weight_mnb", + "test_coef_intercept_shape", + "test_check_accuracy_on_digits", + "test_feature_log_prob_bnb", + "test_bnb", + "test_cnb", + "test_naive_bayes_scale_invariance", + "test_alpha", + "test_alpha_vector" + ], + "types": {} + }, + "test_pipeline": { + "functions": [ + "test_pipeline_init", + "test_pipeline_init_tuple", + "test_pipeline_fit_params", + "test_pipeline_sample_weight_supported", + "test_pipeline_sample_weight_unsupported", + "test_pipeline_raise_set_params_error", + "test_pipeline_methods_pca_svm", + "test_pipeline_methods_preprocessing_svm", + "test_fit_predict_on_pipeline", + "test_fit_predict_on_pipeline_without_fit_predict", + "test_fit_predict_with_intermediate_fit_params", + "test_predict_with_predict_params", + "test_feature_union", + "test_make_union", + "test_make_union_kwargs", + "test_pipeline_transform", + "test_pipeline_fit_transform", + "test_pipeline_slice", + "test_pipeline_index", + "test_set_pipeline_steps", + "test_pipeline_named_steps", + "test_pipeline_ducktyping", + "test_make_pipeline", + "test_feature_union_weights", + "test_feature_union_parallel", + "test_feature_union_feature_names", + "test_set_feature_union_steps", + "test_step_name_validation", + "test_pipeline_wrong_memory", + "test_pipeline_with_cache_attribute", + "test_pipeline_memory", + "test_make_pipeline_memory", + "test_pipeline_param_error" + ], + "types": { + "NoFit": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "NoFit" + } + ] + }, + "NoTrans": { + "methods": [ + "fit", + "get_params", + { + "name": "set_params", + "updates": [0] + } + ] + }, + "NoInvTransf": { + "methods": [ + "transform", + "fit", + "get_params", + { + "name": "set_params", + "updates": [0] + } + ] + }, + "Transf": { + "methods": [ + "transform", + "inverse_transform", + "fit", + "get_params", + { + "name": "set_params", + "updates": [0] + } + ] + }, + "TransfFitParams": { + "methods": [ + { + "name": "fit", + "updates": [0] + }, + "transform", + "inverse_transform", + "get_params", + { + "name": "set_params", + "updates": [0] + } + ] + }, + "Mult": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "Mult" + }, + "fit", + "transform", + "inverse_transform", + "predict", + "score" + ] + }, + "FitParamT": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "FitParamT" + }, + { + "name": "fit", + "updates": [0] + }, + "predict", + { + "name": "fit_predict", + "updates": [0] + }, + "score" + ] + }, + "DummyTransf": { + "methods": [ + { + "name": "fit", + "updates": [0] + }, + "transform", + "inverse_transform", + "get_params", + { + "name": "set_params", + "updates": [0] + } + ] + }, + "DummyEstimatorParams": { + "methods": [ + "fit", + { + "name": "predict", + "updates": [0] + } + ] + }, + "DummyMemory": { + "methods": ["cache"] + }, + "WrongDummyMemory": { + "methods": [] + } + } + }, + "test_random_projection": { + "functions": [ + "make_sparse_random_data", + "densify", + "test_invalid_jl_domain", + "test_input_size_jl_min_dim", + "check_input_size_random_matrix", + "check_size_generated", + "check_zero_mean_and_unit_norm", + "check_input_with_sparse_random_matrix", + "test_gaussian_random_matrix", + "test_sparse_random_matrix", + "test_sparse_random_projection_transformer_invalid_density", + "test_random_projection_transformer_invalid_input", + "test_try_to_transform_before_fit", + "test_too_many_samples_to_find_a_safe_embedding", + "test_random_projection_embedding_quality", + "test_SparseRandomProjection_output_representation", + "test_correct_RandomProjection_dimensions_embedding", + "test_warning_n_components_greater_than_n_features", + "test_works_with_sparse_data" + ], + "types": {} + }, + "test_site_joblib": { + "functions": ["test_old_pickle"], + "types": {} + } + }, + "types": {}, + "functions": [] + }, + "tree": { + "modules": { + "tests": { + "modules": { + "test_export": { + "functions": [ + "test_graphviz_toy", + "test_graphviz_errors", + "test_friedman_mse_in_graphviz", + "test_precision", + "test_export_text_errors", + "test_export_text", + "test_plot_tree_entropy", + "test_plot_tree_gini" + ], + "types": {} + }, + "test_reingold_tilford": { + "functions": [], + "types": {} + }, + "test_tree": { + "functions": [ + "assert_tree_equal", + "test_classification_toy", + "test_weighted_classification_toy", + "test_regression_toy", + "test_xor", + "test_iris", + "test_boston", + "test_probability", + "test_arrayrepr", + "test_pure_set", + "test_numerical_stability", + "test_importances", + "test_importances_raises", + "test_importances_gini_equal_mse", + "test_max_features", + "test_error", + "test_min_samples_split", + "test_min_samples_leaf", + "check_min_weight_fraction_leaf", + "check_min_weight_fraction_leaf_with_min_samples_leaf", + "test_min_impurity_split", + "test_min_impurity_decrease", + "test_multioutput", + "test_classes_shape", + "test_unbalanced_iris", + "test_memory_layout", + "test_sample_weight", + "test_sample_weight_invalid", + "check_class_weights", + "check_class_weight_errors", + "test_max_leaf_nodes", + "test_max_leaf_nodes_max_depth", + "test_arrays_persist", + "test_only_constant_features", + "test_behaviour_constant_feature_after_splits", + "test_with_only_one_non_constant_features", + "test_big_input", + "test_realloc", + "test_huge_allocations", + "check_sparse_input", + "check_sparse_parameters", + "check_sparse_criterion", + "check_explicit_sparse_zeros", + "_check_min_weight_leaf_split_level", + "check_min_weight_leaf_split_level", + "check_public_apply", + "check_public_apply_sparse", + "check_presort_sparse", + "test_presort_sparse", + "test_decision_path_hardcoded", + "check_decision_path", + "check_no_sparse_y_support", + "test_mae", + "test_criterion_copy", + "test_empty_leaf_infinite_threshold", + "test_decision_tree_memmap" + ], + "types": {} + } + }, + "types": {}, + "functions": [] + }, + "export": { + "functions": [ + "_color_brew", + "plot_tree", + "export_graphviz", + "_compute_depth", + "export_text" + ], + "types": { + "Sentinel": { + "methods": ["__repr__"] + }, + "_BaseTreeExporter": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "_BaseTreeExporter" + }, + "get_color", + "get_fill_color", + "node_to_str" + ] + }, + "_DOTTreeExporter": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "_DOTTreeExporter" + }, + "export", + "tail", + "head", + "recurse", + "get_color", + "get_fill_color", + "node_to_str" + ] + }, + "_MPLTreeExporter": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "_MPLTreeExporter" + }, + "_make_tree", + "export", + "recurse", + "get_color", + "get_fill_color", + "node_to_str" + ] + } + } + }, + "setup": { + "functions": ["configuration"], + "types": {} + }, + "tree": { + "functions": [], + "types": { + "BaseDecisionTree": { + "methods": [ + "get_depth", + "get_n_leaves", + { + "name": "fit", + "updates": [0] + }, + "_validate_X_predict", + "predict", + "apply", + "decision_path" + ] + }, + "DecisionTreeClassifier": { + "methods": [ + { + "name": "__init__", + "updates": [], + "returns": "DecisionTreeClassifier" + }, + "__init__", + { + "name": "fit", + "updates": [0] + }, + "predict_proba", + "predict_log_proba", + "get_depth", + "get_n_leaves", + "predict", + "apply", + "decision_path" + ] + }, + "DecisionTreeRegressor": { + "methods": [ + { + "name": "__init__", + "updates": [], + "returns": "DecisionTreeRegressor" + }, + "__init__", + { + "name": "fit", + "updates": [0] + }, + "get_depth", + "get_n_leaves", + "predict", + "apply", + "decision_path" + ] + }, + "ExtraTreeClassifier": { + "methods": [ + { + "name": "__init__", + "updates": [], + "returns": "ExtraTreeClassifier" + }, + "__init__", + { + "name": "fit", + "updates": [0] + }, + "predict_proba", + "predict_log_proba", + "get_depth", + "get_n_leaves", + "predict", + "apply", + "decision_path" + ] + }, + "ExtraTreeRegressor": { + "methods": [ + { + "name": "__init__", + "updates": [], + "returns": "ExtraTreeRegressor" + }, + "__init__", + { + "name": "fit", + "updates": [0] + }, + "get_depth", + "get_n_leaves", + "predict", + "apply", + "decision_path" + ] + } + } + } + }, + "types": { + "DecisionTreeClassifier": { + "methods": [ + { + "name": "__init__", + "updates": [], + "returns": "DecisionTreeClassifier" + }, + "__init__", + { + "name": "fit", + "updates": [0] + }, + "predict_proba", + "predict_log_proba", + "get_depth", + "get_n_leaves", + "predict", + "apply", + "decision_path" + ] + }, + "DecisionTreeRegressor": { + "methods": [ + { + "name": "__init__", + "updates": [], + "returns": "DecisionTreeRegressor" + }, + "__init__", + { + "name": "fit", + "updates": [0] + }, + "get_depth", + "get_n_leaves", + "predict", + "apply", + "decision_path" + ] + }, + "ExtraTreeClassifier": { + "methods": [ + { + "name": "__init__", + "updates": [], + "returns": "ExtraTreeClassifier" + }, + "__init__", + { + "name": "fit", + "updates": [0] + }, + "predict_proba", + "predict_log_proba", + "get_depth", + "get_n_leaves", + "predict", + "apply", + "decision_path" + ] + }, + "ExtraTreeRegressor": { + "methods": [ + { + "name": "__init__", + "updates": [], + "returns": "ExtraTreeRegressor" + }, + "__init__", + { + "name": "fit", + "updates": [0] + }, + "get_depth", + "get_n_leaves", + "predict", + "apply", + "decision_path" + ] + } + }, + "functions": ["export_graphviz", "plot_tree", "export_text"] + }, + "utils": { + "modules": { + "tests": { + "modules": { + "test_class_weight": { + "functions": [ + "test_compute_class_weight", + "test_compute_class_weight_not_present", + "test_compute_class_weight_dict", + "test_compute_class_weight_balanced_negative", + "test_compute_class_weight_balanced_unordered", + "test_compute_class_weight_default", + "test_compute_sample_weight", + "test_compute_sample_weight_with_subsample", + "test_compute_sample_weight_errors", + "test_compute_sample_weight_more_than_32" + ], + "types": {} + }, + "test_cython_blas": { + "functions": ["_no_op"], + "types": {} + }, + "test_deprecation": { + "functions": [ + "test_deprecated", + "test_is_deprecated", + "test_pickle" + ], + "types": { + "MockClass1": { + "methods": [] + }, + "MockClass2": { + "methods": [] + }, + "MockClass3": { + "methods": [] + }, + "MockClass4": { + "methods": [] + } + } + }, + "test_estimator_checks": { + "functions": [ + "test_check_fit_score_takes_y_works_on_deprecated_fit", + "test_check_estimator", + "test_check_outlier_corruption", + "test_check_estimator_transformer_no_mixin", + "test_check_estimator_clones", + "test_check_estimators_unfitted", + "test_check_no_attributes_set_in_init", + "test_check_estimator_pairwise", + "run_tests_without_pytest", + "test_check_class_weight_balanced_linear_classifier" + ], + "types": { + "CorrectNotFittedError": { + "methods": [] + }, + "BaseBadClassifier": { + "methods": ["fit", "predict"] + }, + "ChangesDict": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "ChangesDict" + }, + "fit", + { + "name": "predict", + "updates": [0] + } + ] + }, + "SetsWrongAttribute": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "SetsWrongAttribute" + }, + { + "name": "fit", + "updates": [0] + } + ] + }, + "ChangesWrongAttribute": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "ChangesWrongAttribute" + }, + { + "name": "fit", + "updates": [0] + } + ] + }, + "ChangesUnderscoreAttribute": { + "methods": [ + { + "name": "fit", + "updates": [0] + } + ] + }, + "RaisesErrorInSetParams": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "RaisesErrorInSetParams" + }, + { + "name": "set_params", + "updates": [0] + }, + "fit" + ] + }, + "ModifiesValueInsteadOfRaisingError": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "ModifiesValueInsteadOfRaisingError" + }, + { + "name": "set_params", + "updates": [0] + }, + "fit" + ] + }, + "ModifiesAnotherValue": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "ModifiesAnotherValue" + }, + { + "name": "set_params", + "updates": [0] + }, + "fit" + ] + }, + "NoCheckinPredict": { + "methods": ["fit", "predict"] + }, + "NoSparseClassifier": { + "methods": ["fit", "predict"] + }, + "CorrectNotFittedErrorClassifier": { + "methods": [ + { + "name": "fit", + "updates": [0] + }, + "predict" + ] + }, + "NoSampleWeightPandasSeriesType": { + "methods": ["fit", "predict"] + }, + "BadBalancedWeightsClassifier": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "BadBalancedWeightsClassifier" + }, + { + "name": "fit", + "updates": [0] + }, + "predict" + ] + }, + "BadTransformerWithoutMixin": { + "methods": ["fit", "transform"] + }, + "NotInvariantPredict": { + "methods": ["fit", "predict"] + }, + "LargeSparseNotSupportedClassifier": { + "methods": ["fit"] + }, + "SparseTransformer": { + "methods": [ + { + "name": "fit", + "updates": [0] + }, + { + "name": "fit_transform", + "updates": [0] + }, + "transform" + ] + }, + "TestEstimatorWithDeprecatedFitMethod": { + "methods": [] + }, + "NonConformantEstimatorPrivateSet": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "NonConformantEstimatorPrivateSet" + } + ] + }, + "NonConformantEstimatorNoParamSet": { + "methods": [ + { + "name": "__init__", + "updates": [], + "returns": "NonConformantEstimatorNoParamSet" + }, + "__init__" + ] + } + } + }, + "test_extmath": { + "functions": [ + "test_density", + "test_uniform_weights", + "test_random_weights", + "check_randomized_svd_low_rank", + "test_randomized_svd_low_rank_with_noise", + "test_randomized_svd_infinite_rank", + "test_randomized_svd_transpose_consistency", + "test_randomized_svd_power_iteration_normalizer", + "test_randomized_svd_sparse_warnings", + "test_svd_flip", + "test_randomized_svd_sign_flip", + "test_randomized_svd_sign_flip_with_transpose", + "test_cartesian", + "test_logistic_sigmoid", + "test_incremental_variance_update_formulas", + "test_incremental_mean_and_variance_ignore_nan", + "test_incremental_variance_ddof", + "test_vector_sign_flip", + "test_softmax", + "test_stable_cumsum" + ], + "types": {} + }, + "test_fast_dict": { + "functions": [ + "test_int_float_dict", + "test_int_float_dict_argmin" + ], + "types": {} + }, + "test_fixes": { + "functions": ["test_masked_array_obj_dtype_pickleable"], + "types": {} + }, + "test_linear_assignment": { + "functions": [], + "types": {} + }, + "test_metaestimators": { + "functions": [ + "test_delegated_docstring", + "test_if_delegate_has_method" + ], + "types": { + "Prefix": { + "methods": ["func"] + }, + "MockMetaEstimator": { + "methods": [] + }, + "MetaEst": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "MetaEst" + } + ] + }, + "MetaEstTestTuple": { + "methods": [] + }, + "MetaEstTestList": { + "methods": [] + }, + "HasPredict": { + "methods": ["predict"] + }, + "HasNoPredict": { + "methods": [] + } + } + }, + "test_multiclass": { + "functions": [ + "test_unique_labels", + "test_unique_labels_non_specific", + "test_unique_labels_mixed_types", + "test_is_multilabel", + "test_check_classification_targets", + "test_type_of_target", + "test_class_distribution", + "test_safe_split_with_precomputed_kernel", + "test_ovr_decision_function" + ], + "types": { + "NotAnArray": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "NotAnArray" + }, + "__array__" + ] + } + } + }, + "test_murmurhash": { + "functions": [ + "test_mmhash3_int", + "test_mmhash3_int_array", + "test_mmhash3_bytes", + "test_mmhash3_unicode", + "test_no_collision_on_byte_range", + "test_uniform_distribution" + ], + "types": {} + }, + "test_optimize": { + "functions": ["test_newton_cg"], + "types": {} + }, + "test_pprint": { + "functions": [ + "test_basic", + "test_changed_only", + "test_pipeline", + "test_deeply_nested", + "test_gridsearch", + "test_gridsearch_pipeline", + "test_n_max_elements_to_show", + "test_bruteforce_ellipsis", + "test_builtin_prettyprinter" + ], + "types": { + "LogisticRegression": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "LogisticRegression" + }, + { "name": "fit", "updates": [0] }, + "score", + "predict", + "predict_proba", + "predict_log_proba" + ] + }, + "StandardScaler": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "StandardScaler" + }, + "transform" + ] + }, + "RFE": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "RFE" + } + ] + }, + "GridSearchCV": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "GridSearchCV" + } + ] + }, + "CountVectorizer": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "CountVectorizer" + } + ] + }, + "Pipeline": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "Pipeline" + } + ] + }, + "SVC": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "SVC" + } + ] + }, + "PCA": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "PCA" + } + ] + }, + "NMF": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "NMF" + } + ] + }, + "SimpleImputer": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "SimpleImputer" + } + ] + } + } + }, + "test_random": { + "functions": [ + "test_invalid_sample_without_replacement_algorithm", + "test_sample_without_replacement_algorithms", + "check_edge_case_of_sample_int", + "check_sample_int", + "check_sample_int_distribution", + "test_random_choice_csc", + "test_random_choice_csc_errors", + "test_our_rand_r" + ], + "types": {} + }, + "test_seq_dataset": { + "functions": [ + "assert_csr_equal_values", + "make_dense_dataset_32", + "make_dense_dataset_64", + "make_sparse_dataset_32", + "make_sparse_dataset_64", + "test_buffer_dtype_mismatch_error" + ], + "types": {} + }, + "test_shortest_path": { + "functions": [ + { + "name": "floyd_warshall_slow", + "updates": [0] + }, + "generate_graph", + "test_floyd_warshall", + "test_dijkstra", + "test_shortest_path", + "test_dijkstra_bug_fix" + ], + "types": {} + }, + "test_show_versions": { + "functions": [ + "test_get_sys_info", + "test_get_deps_info", + "test_show_versions_with_blas" + ], + "types": {} + }, + "test_sparsefuncs": { + "functions": [ + "test_mean_variance_axis0", + "test_mean_variance_axis1", + "test_incr_mean_variance_axis", + "test_mean_variance_illegal_axis", + "test_densify_rows", + "test_inplace_column_scale", + "test_inplace_row_scale", + "test_inplace_swap_row", + "test_inplace_swap_column", + "test_min_max_axis_errors", + "test_count_nonzero", + "test_csc_row_median", + "test_inplace_normalize" + ], + "types": {} + }, + "test_testing": { + "functions": [ + "test_assert_less", + "test_assert_greater", + "test_assert_less_equal", + "test_assert_greater_equal", + "test_set_random_state", + "test_assert_allclose_dense_sparse", + "test_assert_raises_msg", + "test_assert_raise_message", + "test_ignore_warning", + "f_ok", + "f_bad_sections", + "f_bad_order", + "f_missing", + "f_check_param_definition", + "test_check_docstring_parameters", + "check_memmap", + "test_tempmemmap", + "test_create_memmap_backed_data" + ], + "types": { + "TestWarns": { + "methods": ["test_warn", "test_warn_wrong_warning"] + }, + "Klass": { + "methods": ["f_missing", "f_bad_sections"] + }, + "MockEst": { + "methods": [ + { + "name": "__init__", + "updates": [], + "returns": "MockEst" + }, + "__init__", + "fit", + "predict", + "predict_proba", + "score" + ] + }, + "MockMetaEstimator": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "MockMetaEstimator" + } + ] + }, + "RegistrationCounter": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "RegistrationCounter" + }, + { + "name": "__call__", + "updates": [0] + } + ] + } + } + }, + "test_utils": { + "functions": [ + "test_make_rng", + "test_deprecated", + "test_resample", + "test_resample_stratified", + "test_resample_stratified_replace", + "test_resample_stratify_2dy", + "test_resample_stratify_sparse_error", + "test_safe_mask", + "test_column_or_1d", + "test_safe_indexing", + "test_safe_indexing_pandas", + "test_safe_indexing_mock_pandas", + "test_shuffle_on_ndim_equals_three", + "test_shuffle_dont_convert_to_array", + "test_gen_even_slices", + "dummy_func", + "test_deprecation_joblib_api" + ], + "types": { + "Ham": { + "methods": [] + } + } + }, + "test_validation": { + "functions": [ + "test_as_float_array", + "test_np_matrix", + "test_memmap", + "test_ordering", + "test_check_array_force_all_finite_object", + "test_check_array_pandas_dtype_object_conversion", + "test_check_array_on_mock_dataframe", + "test_check_array_dtype_stability", + "test_check_array_dtype_warning", + "test_check_array_warn_on_dtype_deprecation", + "test_check_array_accept_sparse_type_exception", + "test_check_array_accept_sparse_no_exception", + "test_check_array_accept_large_sparse_no_exception", + "test_check_array_accept_large_sparse_raise_exception", + "test_check_array_min_samples_and_features_messages", + "test_check_array_complex_data_error", + "test_has_fit_parameter", + "test_check_symmetric", + "test_check_is_fitted", + "test_check_consistent_length", + "test_check_dataframe_fit_attribute", + "test_suppress_validation", + "test_check_array_series", + "test_check_dataframe_warns_on_dtype", + "test_check_X_y_informative_error", + "test_retrieve_samples_from_non_standard_shape" + ], + "types": { + "TestClassWithDeprecatedFitMethod": { + "methods": [] + }, + "DummyMemory": { + "methods": ["cache"] + }, + "WrongDummyMemory": { + "methods": [] + }, + "TestNonNumericShape": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "TestNonNumericShape" + }, + "__len__" + ] + } + } + } + }, + "types": {}, + "functions": [] + }, + "class_weight": { + "functions": ["compute_class_weight", "compute_sample_weight"], + "types": {} + }, + "deprecation": { + "functions": ["_is_deprecated"], + "types": { + "deprecated": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "deprecated" + }, + { + "name": "__call__", + "updates": [1] + }, + { + "name": "_decorate_class", + "updates": [1] + }, + "_decorate_fun", + "_decorate_property", + "_update_doc" + ] + } + } + }, + "estimator_checks": { + "functions": [ + "_safe_tags", + "_yield_checks", + "_yield_classifier_checks", + "_yield_regressor_checks", + "_yield_transformer_checks", + "_yield_clustering_checks", + "_yield_outliers_checks", + "_yield_all_checks", + "check_estimator", + "_boston_subset", + { + "name": "set_checking_parameters", + "updates": [0] + }, + "_is_pairwise", + "_is_pairwise_metric", + "pairwise_estimator_convert_X", + "_generate_sparse_matrix", + "check_estimator_sparse_data", + "check_complex_data", + "is_public_parameter", + "_apply_on_subsets", + "_check_transformer", + "check_outlier_corruption", + "check_outliers_train", + "choose_check_classifiers_labels", + "check_classifiers_classes", + "check_no_attributes_set_in_init", + "check_parameters_default_constructible", + "multioutput_estimator_convert_y_2d", + "check_outliers_fit_predict", + "check_fit_idempotent" + ], + "types": { + "NotAnArray": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "NotAnArray" + }, + "__array__" + ] + } + } + }, + "extmath": { + "functions": [ + "squared_norm", + "row_norms", + "fast_logdet", + "density", + "safe_sparse_dot", + "randomized_range_finder", + "randomized_svd", + "weighted_mode", + { + "name": "cartesian", + "updates": [1] + }, + "svd_flip", + "log_logistic", + "softmax", + "safe_min", + "make_nonnegative", + "_safe_accumulator_op", + "_incremental_mean_and_var", + "_deterministic_vector_sign_flip", + "stable_cumsum" + ], + "types": {} + }, + "fixes": { + "functions": [ + "_parse_version", + "parallel_helper", + "_astype_copy_false", + "_joblib_parallel_args" + ], + "types": { + "MaskedArray": { + "methods": ["__getstate__"] + } + } + }, + "graph": { + "functions": ["single_source_shortest_path_length"], + "types": {} + }, + "linear_assignment_": { + "functions": [ + "linear_assignment", + "_hungarian", + { + "name": "_step1", + "updates": [0] + }, + "_step3", + { + "name": "_step4", + "updates": [0] + }, + "_step5", + "_step6" + ], + "types": { + "_HungarianState": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "_HungarianState" + }, + "_clear_covers" + ] + } + } + }, + "metaestimators": { + "functions": ["if_delegate_has_method", "_safe_split"], + "types": { + "_BaseComposition": { + "methods": [ + "_get_params", + "_set_params", + "_replace_estimator", + "_validate_names" + ] + }, + "_IffHasAttrDescriptor": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "_IffHasAttrDescriptor" + }, + "__get__" + ] + } + } + }, + "mocking": { + "functions": [], + "types": { + "ArraySlicingWrapper": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "ArraySlicingWrapper" + }, + "__getitem__" + ] + }, + "MockDataFrame": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "MockDataFrame" + }, + "__len__", + "__array__", + "__eq__", + "__ne__" + ] + }, + "CheckingClassifier": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "CheckingClassifier" + }, + { + "name": "fit", + "updates": [0] + }, + "predict", + "score", + "_more_tags" + ] + } + } + }, + "multiclass": { + "functions": [ + "_unique_multiclass", + "_unique_indicator", + "unique_labels", + "_is_integral_float", + "is_multilabel", + "check_classification_targets", + "type_of_target", + { + "name": "_check_partial_fit_first_call", + "updates": [0] + }, + "class_distribution", + "_ovr_decision_function" + ], + "types": {} + }, + "optimize": { + "functions": ["_line_search_wolfe12", "_cg", "newton_cg"], + "types": { + "_LineSearchError": { + "methods": [] + } + } + }, + "random": { + "functions": [ + { + "name": "random_choice_csc", + "updates": [1] + } + ], + "types": {} + }, + "setup": { + "functions": ["configuration"], + "types": {} + }, + "sparsefuncs": { + "functions": [ + "_raise_typeerror", + "_raise_error_wrong_axis", + { + "name": "inplace_csr_column_scale", + "updates": [0] + }, + { + "name": "inplace_csr_row_scale", + "updates": [0] + }, + "mean_variance_axis", + "incr_mean_variance_axis", + "inplace_column_scale", + "inplace_row_scale", + "inplace_swap_row_csc", + { + "name": "inplace_swap_row_csr", + "updates": [0] + }, + "inplace_swap_row", + "inplace_swap_column", + "_minor_reduce", + "_min_or_max_axis", + "_sparse_min_or_max", + "_sparse_min_max", + "_sparse_nan_min_max", + "min_max_axis", + "count_nonzero", + "_get_median", + "_get_elem_at_rank", + "csc_median_axis_0" + ], + "types": {} + }, + "stats": { + "functions": ["_weighted_percentile"], + "types": {} + }, + "testing": { + "functions": [ + "assert_warns", + "assert_warns_message", + "assert_warns_div0", + "assert_no_warnings", + "ignore_warnings", + "assert_raise_message", + "assert_allclose_dense_sparse", + "install_mldata_mock", + "uninstall_mldata_mock", + "all_estimators", + "set_random_state", + "clean_warning_registry", + "check_skip_network", + "_delete_folder", + "create_memmap_backed_data", + "_get_args", + "_get_func_name", + "check_docstring_parameters", + "assert_run_python_script" + ], + "types": { + "_IgnoreWarnings": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "_IgnoreWarnings" + }, + "__call__", + "__repr__", + { + "name": "__enter__", + "updates": [0] + }, + "__exit__" + ] + }, + "mock_mldata_urlopen": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "mock_mldata_urlopen" + }, + "__call__" + ] + }, + "TempMemmap": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "TempMemmap" + }, + { + "name": "__enter__", + "updates": [0] + }, + "__exit__" + ] + } + } + }, + "validation": { + "functions": [ + "_assert_all_finite", + "assert_all_finite", + "as_float_array", + "_is_arraylike", + "_num_samples", + "check_memory", + "check_consistent_length", + "indexable", + "_ensure_sparse_format", + "_ensure_no_complex_data", + "check_array", + "_check_large_sparse", + "check_X_y", + "column_or_1d", + "check_random_state", + "has_fit_parameter", + "check_symmetric", + "check_is_fitted", + "check_non_negative", + "check_scalar" + ], + "types": {} + } + }, + "types": { + "deprecated": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "deprecated" + }, + { + "name": "__call__", + "updates": [1] + }, + { + "name": "_decorate_class", + "updates": [1] + }, + "_decorate_fun", + "_decorate_property", + "_update_doc" + ] + } + }, + "functions": [ + "compute_class_weight", + "compute_sample_weight", + "as_float_array", + "assert_all_finite", + "check_random_state", + "column_or_1d", + "check_array", + "check_consistent_length", + "check_X_y", + "indexable", + "check_symmetric", + "check_scalar" + ] + }, + "base": { + "functions": [ + "clone", + "_pprint", + "_update_if_consistent", + "is_classifier", + "is_regressor", + "is_outlier_detector" + ], + "types": { + "BaseEstimator": { + "methods": [ + "get_params", + "set_params", + "__repr__", + "__getstate__", + "__setstate__", + "_get_tags" + ] + }, + "ClassifierMixin": { + "methods": ["score"] + }, + "RegressorMixin": { + "methods": ["score"] + }, + "ClusterMixin": { + "methods": ["fit_predict"] + }, + "BiclusterMixin": { + "methods": ["get_indices", "get_shape", "get_submatrix"] + }, + "TransformerMixin": { + "methods": ["fit_transform"] + }, + "DensityMixin": { + "methods": ["score"] + }, + "OutlierMixin": { + "methods": ["fit_predict"] + }, + "MetaEstimatorMixin": { + "methods": [] + }, + "MultiOutputMixin": { + "methods": ["_more_tags"] + }, + "_UnstableArchMixin": { + "methods": ["_more_tags"] + } + } + }, + "calibration": { + "functions": ["_sigmoid_calibration", "calibration_curve"], + "types": { + "CalibratedClassifierCV": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "CalibratedClassifierCV" + }, + { + "name": "fit", + "updates": [0] + }, + "predict_proba", + "predict" + ] + }, + "_CalibratedClassifier": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "_CalibratedClassifier" + }, + "_preproc", + { + "name": "fit", + "updates": [0] + }, + "predict_proba" + ] + }, + "_SigmoidCalibration": { + "methods": [ + { + "name": "fit", + "updates": [0] + }, + "predict" + ] + } + } + }, + "conftest": { + "functions": [], + "types": {} + }, + "discriminant_analysis": { + "functions": ["_cov", "_class_means", "_class_cov"], + "types": { + "LinearDiscriminantAnalysis": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "LinearDiscriminantAnalysis" + }, + { + "name": "_solve_lsqr", + "updates": [0] + }, + { + "name": "_solve_eigen", + "updates": [0] + }, + { + "name": "_solve_svd", + "updates": [0] + }, + { + "name": "fit", + "updates": [0] + }, + "transform", + "predict_proba", + "predict_log_proba" + ] + }, + "QuadraticDiscriminantAnalysis": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "QuadraticDiscriminantAnalysis" + }, + { + "name": "fit", + "updates": [0] + }, + "_decision_function", + "decision_function", + "predict", + "predict_proba", + "predict_log_proba" + ] + } + } + }, + "dummy": { + "functions": [], + "types": { + "DummyClassifier": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "DummyClassifier" + }, + { + "name": "fit", + "updates": [0] + }, + "predict", + "predict_proba", + "predict_log_proba", + "_more_tags", + "score" + ] + }, + "DummyRegressor": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "DummyRegressor" + }, + { + "name": "fit", + "updates": [0] + }, + "predict", + "_more_tags", + "score" + ] + } + } + }, + "exceptions": { + "functions": [], + "types": { + "NotFittedError": { + "methods": [] + }, + "ChangedBehaviorWarning": { + "methods": [] + }, + "ConvergenceWarning": { + "methods": [] + }, + "DataConversionWarning": { + "methods": [] + }, + "DataDimensionalityWarning": { + "methods": [] + }, + "EfficiencyWarning": { + "methods": [] + }, + "FitFailedWarning": { + "methods": [] + }, + "NonBLASDotWarning": { + "methods": [] + }, + "SkipTestWarning": { + "methods": [] + }, + "UndefinedMetricWarning": { + "methods": [] + } + } + }, + "isotonic": { + "functions": ["check_increasing", "isotonic_regression"], + "types": { + "IsotonicRegression": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "IsotonicRegression" + }, + "_check_fit_data", + { + "name": "_build_f", + "updates": [0] + }, + { + "name": "_build_y", + "updates": [0] + }, + { + "name": "fit", + "updates": [0] + }, + "transform", + "predict", + "__getstate__", + { + "name": "__setstate__", + "updates": [0] + }, + "_more_tags" + ] + } + } + }, + "kernel_approximation": { + "functions": [], + "types": { + "RBFSampler": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "RBFSampler" + }, + { + "name": "fit", + "updates": [0] + }, + "transform" + ] + }, + "SkewedChi2Sampler": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "SkewedChi2Sampler" + }, + { + "name": "fit", + "updates": [0] + }, + "transform" + ] + }, + "AdditiveChi2Sampler": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "AdditiveChi2Sampler" + }, + { + "name": "fit", + "updates": [0] + }, + "transform", + "_transform_dense", + "_transform_sparse", + "_more_tags" + ] + }, + "Nystroem": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "Nystroem" + }, + { + "name": "fit", + "updates": [0] + }, + "transform", + "_get_kernel_params" + ] + } + } + }, + "kernel_ridge": { + "functions": [], + "types": { + "KernelRidge": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "KernelRidge" + }, + "_get_kernel", + { + "name": "fit", + "updates": [0] + }, + "predict" + ] + } + } + }, + "multiclass": { + "functions": [ + "_fit_binary", + "_partial_fit_binary", + "_predict_binary", + "_check_estimator", + "_fit_ovo_binary", + "_partial_fit_ovo_binary" + ], + "types": { + "_ConstantPredictor": { + "methods": [ + { + "name": "fit", + "updates": [0] + }, + "predict", + "decision_function", + "predict_proba" + ] + }, + "OneVsRestClassifier": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "OneVsRestClassifier" + }, + { + "name": "fit", + "updates": [0] + }, + "predict" + ] + }, + "OneVsOneClassifier": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "OneVsOneClassifier" + }, + { + "name": "fit", + "updates": [0] + }, + "predict", + "decision_function" + ] + }, + "OutputCodeClassifier": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "OutputCodeClassifier" + }, + { + "name": "fit", + "updates": [0] + }, + "predict" + ] + } + } + }, + "multioutput": { + "functions": ["_fit_estimator", "_partial_fit_estimator"], + "types": { + "MultiOutputEstimator": { + "methods": [ + { + "name": "fit", + "updates": [0] + }, + "predict", + "_more_tags" + ] + }, + "MultiOutputRegressor": { + "methods": [ + { + "name": "__init__", + "updates": [], + "returns": "MultiOutputRegressor" + }, + "__init__", + "score", + { + "name": "fit", + "updates": [0] + }, + "predict" + ] + }, + "MultiOutputClassifier": { + "methods": [ + { + "name": "__init__", + "updates": [], + "returns": "MultiOutputClassifier" + }, + "__init__", + "predict_proba", + "score", + "_more_tags", + { + "name": "fit", + "updates": [0] + }, + "predict" + ] + }, + "_BaseChain": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "_BaseChain" + }, + "predict" + ] + }, + "ClassifierChain": { + "methods": [ + { + "name": "fit", + "updates": [0] + }, + "_more_tags", + "predict" + ] + }, + "RegressorChain": { + "methods": ["fit", "_more_tags", "predict"] + } + } + }, + "naive_bayes": { + "functions": [], + "types": { + "BaseNB": { + "methods": ["predict", "predict_log_proba", "predict_proba"] + }, + "GaussianNB": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "GaussianNB" + }, + { + "name": "fit", + "updates": [0] + }, + { + "name": "partial_fit", + "updates": [0] + }, + { + "name": "_partial_fit", + "updates": [0] + }, + "_joint_log_likelihood", + "predict", + "predict_log_proba", + "predict_proba" + ] + }, + "BaseDiscreteNB": { + "methods": [ + { + "name": "_update_class_log_prior", + "updates": [0] + }, + "_check_alpha", + { + "name": "partial_fit", + "updates": [0] + }, + { + "name": "fit", + "updates": [0] + }, + "_get_coef", + "_get_intercept", + "_more_tags", + "predict", + "predict_log_proba", + "predict_proba" + ] + }, + "MultinomialNB": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "MultinomialNB" + }, + { + "name": "_count", + "updates": [0] + }, + { + "name": "_update_feature_log_prob", + "updates": [0] + }, + "_joint_log_likelihood", + { + "name": "partial_fit", + "updates": [0] + }, + { + "name": "fit", + "updates": [0] + }, + "predict", + "predict_log_proba", + "predict_proba" + ] + }, + "ComplementNB": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "ComplementNB" + }, + { + "name": "_count", + "updates": [0] + }, + { + "name": "_update_feature_log_prob", + "updates": [0] + }, + "_joint_log_likelihood", + { + "name": "partial_fit", + "updates": [0] + }, + { + "name": "fit", + "updates": [0] + }, + "predict", + "predict_log_proba", + "predict_proba" + ] + }, + "BernoulliNB": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "BernoulliNB" + }, + { + "name": "_count", + "updates": [0] + }, + { + "name": "_update_feature_log_prob", + "updates": [0] + }, + "_joint_log_likelihood", + { + "name": "partial_fit", + "updates": [0] + }, + { + "name": "fit", + "updates": [0] + }, + "predict", + "predict_log_proba", + "predict_proba" + ] + } + } + }, + "pipeline": { + "functions": [ + "_name_estimators", + "make_pipeline", + "_transform_one", + "_fit_transform_one", + "_fit_one", + "make_union" + ], + "types": { + "Pipeline": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "Pipeline" + }, + "get_params", + "set_params", + "_validate_steps", + "_iter", + "__len__", + "__getitem__", + "_log_message", + { + "name": "_fit", + "updates": [0] + }, + { + "name": "fit", + "updates": [0] + }, + { + "name": "fit_transform", + "updates": [0] + }, + "_transform", + "_inverse_transform" + ] + }, + "FeatureUnion": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "FeatureUnion" + }, + "get_params", + "set_params", + "_validate_transformers", + "_iter", + "get_feature_names", + { + "name": "fit", + "updates": [0] + }, + { + "name": "fit_transform", + "updates": [0] + }, + "_log_message", + { + "name": "_parallel_func", + "updates": [0] + }, + "transform", + "_update_transformer_list" + ] + } + } + }, + "random_projection": { + "functions": [ + "johnson_lindenstrauss_min_dim", + "_check_density", + "_check_input_size", + "gaussian_random_matrix", + "sparse_random_matrix" + ], + "types": { + "BaseRandomProjection": { + "methods": [ + { + "name": "fit", + "updates": [0] + }, + "transform" + ] + }, + "GaussianRandomProjection": { + "methods": [ + { + "name": "__init__", + "updates": [], + "returns": "GaussianRandomProjection" + }, + "__init__", + "_make_random_matrix", + { + "name": "fit", + "updates": [0] + }, + "transform" + ] + }, + "SparseRandomProjection": { + "methods": [ + { + "name": "__init__", + "updates": [0], + "returns": "SparseRandomProjection" + }, + { + "name": "_make_random_matrix", + "updates": [0] + }, + { + "name": "fit", + "updates": [0] + }, + "transform" + ] + } + } + }, + "setup": { + "functions": ["configuration"], + "types": {} + } + }, + "types": {}, + "functions": [] + } +} diff --git a/types/@msrvida-python-program-analysis/symbol-table.d.ts b/types/@msrvida-python-program-analysis/symbol-table.d.ts new file mode 100644 index 000000000000..dc0f5e9ee204 --- /dev/null +++ b/types/@msrvida-python-program-analysis/symbol-table.d.ts @@ -0,0 +1,26 @@ +import { FunctionSpec, TypeSpec, ModuleSpec, ModuleMap, JsonSpecs } from "./specs"; +import * as ast from './python-parser'; +export declare class SymbolTable { + private jsonSpecs; + modules: ModuleMap; + types: { + [name: string]: TypeSpec; + }; + functions: { + [name: string]: FunctionSpec; + }; + constructor(jsonSpecs: JsonSpecs); + lookupFunction(name: string): FunctionSpec | undefined; + lookupNode(func: ast.SyntaxNode): FunctionSpec; + lookupModuleFunction(modName: string, funcName: string): FunctionSpec | undefined; + importModule(modulePath: string, alias: string): ModuleSpec; + private resolveFunction; + private resolveType; + private makePythonType; + private resolveModule; + importModuleDefinitions(namePath: string, imports: { + path: string; + alias: string; + }[]): string[]; + private lookupSpec; +} diff --git a/types/@msrvida-python-program-analysis/test/analysis.test.d.ts b/types/@msrvida-python-program-analysis/test/analysis.test.d.ts new file mode 100644 index 000000000000..cb0ff5c3b541 --- /dev/null +++ b/types/@msrvida-python-program-analysis/test/analysis.test.d.ts @@ -0,0 +1 @@ +export {}; diff --git a/types/@msrvida-python-program-analysis/test/cellslice.test.d.ts b/types/@msrvida-python-program-analysis/test/cellslice.test.d.ts new file mode 100644 index 000000000000..cb0ff5c3b541 --- /dev/null +++ b/types/@msrvida-python-program-analysis/test/cellslice.test.d.ts @@ -0,0 +1 @@ +export {}; diff --git a/types/@msrvida-python-program-analysis/test/cfg.test.d.ts b/types/@msrvida-python-program-analysis/test/cfg.test.d.ts new file mode 100644 index 000000000000..cb0ff5c3b541 --- /dev/null +++ b/types/@msrvida-python-program-analysis/test/cfg.test.d.ts @@ -0,0 +1 @@ +export {}; diff --git a/types/@msrvida-python-program-analysis/test/graph.test.d.ts b/types/@msrvida-python-program-analysis/test/graph.test.d.ts new file mode 100644 index 000000000000..cb0ff5c3b541 --- /dev/null +++ b/types/@msrvida-python-program-analysis/test/graph.test.d.ts @@ -0,0 +1 @@ +export {}; diff --git a/types/@msrvida-python-program-analysis/test/log-slicer.test.d.ts b/types/@msrvida-python-program-analysis/test/log-slicer.test.d.ts new file mode 100644 index 000000000000..cb0ff5c3b541 --- /dev/null +++ b/types/@msrvida-python-program-analysis/test/log-slicer.test.d.ts @@ -0,0 +1 @@ +export {}; diff --git a/types/@msrvida-python-program-analysis/test/magics-rewriter.test.d.ts b/types/@msrvida-python-program-analysis/test/magics-rewriter.test.d.ts new file mode 100644 index 000000000000..cb0ff5c3b541 --- /dev/null +++ b/types/@msrvida-python-program-analysis/test/magics-rewriter.test.d.ts @@ -0,0 +1 @@ +export {}; diff --git a/types/@msrvida-python-program-analysis/test/merge.test.d.ts b/types/@msrvida-python-program-analysis/test/merge.test.d.ts new file mode 100644 index 000000000000..cb0ff5c3b541 --- /dev/null +++ b/types/@msrvida-python-program-analysis/test/merge.test.d.ts @@ -0,0 +1 @@ +export {}; diff --git a/types/@msrvida-python-program-analysis/test/notebooks/03-feature-engineering-tfidf-expected.json b/types/@msrvida-python-program-analysis/test/notebooks/03-feature-engineering-tfidf-expected.json new file mode 100644 index 000000000000..1b754d376e7a --- /dev/null +++ b/types/@msrvida-python-program-analysis/test/notebooks/03-feature-engineering-tfidf-expected.json @@ -0,0 +1,155 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import pandas as pd\n", + "in_vscode=0\n", + "\n", + "df = pd.read_parquet(\"data/training.parquet\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import numpy as np\n", + "\n", + "np.random.seed(0xc0ffeeee)\n", + "df_samp = df.sample(3)\n", + "" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from sklearn.feature_extraction.text import CountVectorizer\n", + "\n", + "vectorizer = CountVectorizer(token_pattern='(?u)\\\\b[A-Za-z]\\\\w+\\\\b', max_features = 20)\n", + "counts = vectorizer.fit_transform(df_samp[\"text\"])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from sklearn.feature_extraction.text import TfidfTransformer\n", + "\n", + "tfidf_transformer = TfidfTransformer()\n", + "df_tfidf = tfidf_transformer.fit_transform(counts)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from sklearn.feature_extraction.text import HashingVectorizer\n", + "BUCKETS=1024\n", + "\n", + "hv = HashingVectorizer(norm=None, token_pattern='(?u)\\\\b[A-Za-z]\\\\w+\\\\b', n_features=BUCKETS, alternate_sign = False)\n", + "hv\n", + "" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "hvcounts = hv.fit_transform(df[\"text\"])\n", + "hvcounts" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "tfidf_transformer = TfidfTransformer()\n", + "hvdf_tfidf = tfidf_transformer.fit_transform(hvcounts)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#PCA projection so that output can be visualised\n", + "\n", + "import sklearn.decomposition\n", + "\n", + "DIMENSIONS = 2\n", + "\n", + "pca2 = sklearn.decomposition.TruncatedSVD(DIMENSIONS)\n", + "\n", + "pca_a = pca2.fit_transform(hvdf_tfidf)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "pca_df = pd.DataFrame(pca_a, columns=[\"x\", \"y\"])\n", + "pca_df.sample(10)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "plot_data = pd.concat([df.reset_index(), pca_df], axis=1)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import altair as alt\n", + "alt.renderers.enable('nteract')\n", + "\n", + "alt.Chart(plot_data.sample(1000)).encode(x=\"x\", y=\"y\", color=\"label\").mark_point().interactive()\n", + "\n", + "" + ] + } + ], + "nbformat": 4, + "nbformat_minor": 2, + "metadata": { + "language_info": { + "name": "python", + "codemirror_mode": { + "name": "ipython", + "version": 3 + } + }, + "orig_nbformat": 2, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "npconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": 3 + } +} diff --git a/types/@msrvida-python-program-analysis/test/notebooks/03-feature-engineering-tfidf.json b/types/@msrvida-python-program-analysis/test/notebooks/03-feature-engineering-tfidf.json new file mode 100644 index 000000000000..2aa0b2e333ff --- /dev/null +++ b/types/@msrvida-python-program-analysis/test/notebooks/03-feature-engineering-tfidf.json @@ -0,0 +1,404 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "In this notebook we will process the synthetic Austen/food reviews data and convert it into feature vectors. In later notebooks these feature vectors will be the inputs to models which we will train and eventually use to identify spam. \n", + "\n", + "This notebook uses [term frequency-inverse document frequency](https://en.wikipedia.org/wiki/Tf–idf), or tf-idf, to generate feature vectors. Tf-idf is commonly used to summarise text data, and it aims to capture how important different words are within a set of documents. Tf-idf combines a normalized word count (or term frequency) with the inverse document frequency (or a measure of how common a word is across all documents) in order to identify words, or terms, which are 'interesting' or important within the document. \n", + "\n", + "\n", + "We begin by loading in the data:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import pandas as pd\n", + "in_vscode=0\n", + "\n", + "df = pd.read_parquet(\"data/training.parquet\")" + ] + }, + { + "cell_type": "markdown", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "To illustrate the computation of tf-idf vectors we will first implement the method on a sample of three of the documents we just loaded. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import numpy as np\n", + "\n", + "np.random.seed(0xc0ffeeee)\n", + "df_samp = df.sample(3)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "pd.set_option('display.max_colwidth', -1) #ensures that all the text is visible\n", + "df_samp" + ] + }, + { + "cell_type": "markdown", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "We begin by computing the term frequency ('tf') of the words in the three texts above. We use the `token_pattern` parameter to specify that we only want to consider words (no numeric values). We limit the number of words (`max_features`) to 20, so that we can easily inspect the output. This means that only the 20 words which appear most frequently across the three texts will be represented. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from sklearn.feature_extraction.text import CountVectorizer\n", + "\n", + "vectorizer = CountVectorizer(token_pattern='(?u)\\\\b[A-Za-z]\\\\w+\\\\b', max_features = 20)\n", + "counts = vectorizer.fit_transform(df_samp[\"text\"])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "vectorizer.get_feature_names() #shows all the words used as features for this vectorizer" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "counts" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "print(counts.toarray()) " + ] + }, + { + "cell_type": "markdown", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "Each row of the array corresponds to one of the texts, whilst the columns relate to the words considered in this vectorizer. (You can confirm that 'all' appears once in the first two texts, and twice in the third text, and so on.)\n", + "\n", + "The next stage of the process is to use the results of the term frequency matrix to compute the tf-idf. \n", + "\n", + "The inverse document frequency (idf) for a particular word, or feature, is computed as (the log of) a ratio of the number of documents in a corpus to the number of documents which contain that feature (up to some constant factors). " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from sklearn.feature_extraction.text import TfidfTransformer\n", + "\n", + "tfidf_transformer = TfidfTransformer()\n", + "df_tfidf = tfidf_transformer.fit_transform(counts)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "print(df_tfidf.toarray())" + ] + }, + { + "cell_type": "markdown", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "Each row of the object above is the desired tf-idf vector for the relevant document. " + ] + }, + { + "cell_type": "markdown", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "A major disadvantage of using a vectorizer is that it will be dependent upon the dictionary of words it sees when it is 'fit' to the data. As such, if we are presented with a new passage of text and wish to compute a feature vector for for that text we are required to know which word maps to which space of the vector. Keeping track of a dictionary is impractical and will lead to inefficiency. \n", + "\n", + "Furthermore, there are only \"spaces\" in the vectorizer for words that have been seen in the fitting stage. If a new text sample contains a word which was not present when the vectorizer was first fit, there will be no place in the feature vectors to count that word. \n", + "\n", + "With that in mind, we consider using a [hashing vectorizer](https://en.wikipedia.org/wiki/Feature_hashing). Words can be hashed to buckets, and the bucket count incremented. This will give us a counts matrix, like we saw above, which we can then compute the tf-idf matrix for, without the need to keep track of which column in the matrix any given word maps to. \n", + "\n", + "One disadvantage of this approach is that collisions will occur - with a finite set of buckets multiple words will hash to the same bucket. As such we are no longer computing an exact tf-idf matrix.\n", + "\n", + "Furthermore we will not be able to recover the word (or words) associated with a bucket at a later time if we need them. (For our application this won't be needed.)\n", + "\n", + "We fix the number of buckets at 210 = 1024, but you can try using a different number of buckets and see how the spam detection models are effected. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from sklearn.feature_extraction.text import HashingVectorizer\n", + "BUCKETS=1024\n", + "\n", + "hv = HashingVectorizer(norm=None, token_pattern='(?u)\\\\b[A-Za-z]\\\\w+\\\\b', n_features=BUCKETS, alternate_sign = False)\n", + "hv\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "hvcounts = hv.fit_transform(df[\"text\"])\n", + "hvcounts" + ] + }, + { + "cell_type": "markdown", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "We can then go on to compute the \"approximate\" tf-idf matrix for this, by applying the tf-idf transformer to the hashed counts matrix." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "tfidf_transformer = TfidfTransformer()\n", + "hvdf_tfidf = tfidf_transformer.fit_transform(hvcounts)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "hvdf_tfidf" + ] + }, + { + "cell_type": "markdown", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "We apply PCA so that we can visualize the output" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#PCA projection so that output can be visualised\n", + "\n", + "import sklearn.decomposition\n", + "\n", + "DIMENSIONS = 2\n", + "\n", + "pca2 = sklearn.decomposition.TruncatedSVD(DIMENSIONS)\n", + "\n", + "pca_a = pca2.fit_transform(hvdf_tfidf)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "pca_df = pd.DataFrame(pca_a, columns=[\"x\", \"y\"])\n", + "pca_df.sample(10)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "plot_data = pd.concat([df.reset_index(), pca_df], axis=1)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import altair as alt\n", + "alt.renderers.enable('nteract')\n", + "\n", + "alt.Chart(plot_data.sample(1000)).encode(x=\"x\", y=\"y\", color=\"label\").mark_point().interactive()" + ] + }, + { + "cell_type": "markdown", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "We want to be able to easily compute feature vectors using the hashing tf-idf workflow laid out above. The `Pipeline` facility in scikit-learn streamlines this workflow by making it easy to pass data through multiple transforms. In the next cell we set up our pipeline." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from sklearn.feature_extraction.text import HashingVectorizer,TfidfTransformer\n", + "from sklearn.pipeline import Pipeline\n", + "import pickle, os\n", + "\n", + "vect = HashingVectorizer(norm=None, token_pattern='(?u)\\\\b[A-Za-z]\\\\w+\\\\b', n_features=BUCKETS, alternate_sign = False)\n", + "tfidf = TfidfTransformer()\n", + "\n", + "feat_pipeline = Pipeline([\n", + " ('vect',vect),\n", + " ('tfidf',tfidf)\n", + "])" + ] + }, + { + "cell_type": "markdown", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "We can then use the `fit_transform` method to apply the pipeline to our data frame. This produces a sparse matrix (only non zero entries are recorded). We convert this to a dense array using the `toarray()` function, then append the index and labels to aid readability. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "feature_vecs = feat_pipeline.fit_transform(df[\"text\"]).toarray()\n", + "labeled_vecs = pd.concat([df.reset_index()[[\"index\", \"label\"]],\n", + " pd.DataFrame(feature_vecs)], axis=1)\n", + "labeled_vecs.columns = labeled_vecs.columns.astype(str)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "labeled_vecs.sample(10)" + ] + }, + { + "cell_type": "markdown", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "We save the feature vectors to a parquet file." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "labeled_vecs.to_parquet(\"data/features.parquet\")" + ] + }, + { + "cell_type": "markdown", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "We will then serialize our pipeline to a file on disk so that we can reuse the document frequencies we've observed on training data to weight term vectors." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from mlworkflows import util\n", + "util.serialize_to(feat_pipeline, \"feature_pipeline.sav\")" + ] + }, + { + "cell_type": "markdown", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "Now that we have a feature engineering approach, next step is to train a model. Again, you have two choices for your next step: [click here](04-model-logistic-regression.ipynb) for a model based on *logistic regression*, or [click here](04-model-random-forest.ipynb) for a model based on *ensembles of decision trees.*" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3.7.0 64-bit ('ml-workflows': conda)", + "language": "python", + "name": "python37064bitmlworkflowscondad95461eefba54754b428b1d0fa47e7eb" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.0-final" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/types/@msrvida-python-program-analysis/test/notebooks/Pima_Prediction.json b/types/@msrvida-python-program-analysis/test/notebooks/Pima_Prediction.json new file mode 100644 index 000000000000..8a0213f10967 --- /dev/null +++ b/types/@msrvida-python-program-analysis/test/notebooks/Pima_Prediction.json @@ -0,0 +1,1094 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Predicting Diabetes\n", + "## Import Libraries\n" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import pandas as pd # pandas is a dataframe library\n", + "import matplotlib.pyplot as plt # matplotlib.pyplot plots data\n", + "import numpy as np # numpy provides N-dim object support" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Load and review data" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "df = pd.read_csv(\"./data/pima-data.csv\") # load Pima data" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
num_pregglucose_concdiastolic_bpthicknessinsulinbmidiab_predageskindiabetes
061487235033.60.627501.3790True
11856629026.60.351311.1426False
28183640023.30.672320.0000True
318966239428.10.167210.9062False
40137403516843.12.288331.3790True
\n", + "
" + ], + "text/plain": [ + " num_preg glucose_conc diastolic_bp thickness insulin bmi diab_pred \\\n", + "0 6 148 72 35 0 33.6 0.627 \n", + "1 1 85 66 29 0 26.6 0.351 \n", + "2 8 183 64 0 0 23.3 0.672 \n", + "3 1 89 66 23 94 28.1 0.167 \n", + "4 0 137 40 35 168 43.1 2.288 \n", + "\n", + " age skin diabetes \n", + "0 50 1.3790 True \n", + "1 31 1.1426 False \n", + "2 32 0.0000 True \n", + "3 21 0.9062 False \n", + "4 33 1.3790 True " + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df.head(5)" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "def check(df, size=11):\n", + " \"\"\"\n", + " Function plots a graphical correlation matrix for each pair of columns in the dataframe.\n", + " \n", + " Input:\n", + " df: pandas DataFrame\n", + " size: vertical and horizontal size of the plot\n", + " \n", + " Displays:\n", + " matrix of correlation between columns. Blue-cyan-yellow-red-darkred => less to more correlated\n", + " 0------------------------->1\n", + " Expect a darkred line running from top to bottom right\n", + " \"\"\"\n", + " \n", + " corr = df.corr() # data frame correlation function\n", + " fig, ax = plt.subplots(figsize=(size, size))\n", + " ax.matshow(corr) # color code the rectangles by correlation value\n", + " plt.xticks(range(len(corr.columns)), corr.columns) # draw x tick marks\n", + " plt.yticks(range(len(corr.columns)), corr.columns) # draw y tick marks\n", + " \n" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAArEAAAJ5CAYAAAC5YWj0AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzs3Xm4XWV99//3hwRMCJOItaJotIIWUUECFQSNY2vVRxT84axUQRyrfWifWpFSLa1WOylOgSLOtYoojlCRyCgQMISgon0kPlLaCso8C9/fH2sd2R7OlORk79w579d1nStrr3Wve33XHtb57Hvf+yRVhSRJktSSzUZdgCRJkrS2DLGSJElqjiFWkiRJzTHESpIkqTmGWEmSJDXHECtJkqTmGGIlSZLUHEPsHJJk/gbo88QkB812vxuzJEcnOSLJO5M8fR32X5pk35kep19e62MleVWSY9e2vin62y7J6/vlpUm+Okm745PsOkU/vz6vliU5d5b7W5xkdb+8JMn7Z7P/DWWw7nXYd8ckX5jtmtayhhm/npMsT7JkWLWNO/asvp41M0nWJNlhgvWz+vrfEDbkczvJ7kn+cP2rXD+zHmp0jySLgW8AZwP7Av8JPK9fd0RVrehfHCuqanGSVwEHAPOA3YC/B7YAXg7cDvxhVf1ykmMtB1YCewPbAH9UVRckORrYEVgMXJPk5cC7gaXAfYAPVtVHk2wGHAs8GbiC7g3OCVU10l8wG7OqOmodd10K3ATM+CK4HseaTdsBrwc+NFWjqnrNcMoZraqa9o3IevS9AlixofrfWFTVVcBG8SZ4FK+xJPOq6q5hH1frb0O+/mfbBnpu7w4sAb6+AfqeMUdiN7yd6YLio4HrgAOnab8b8BK6MHoMcEtV7QGcB7ximn0X9S+s1wMnDKzfE3heVb0EeDVwfVXtBewFHJrkYcAL6ILuY4DXAPuM7zzJO5L8MMm/J/ns+NG0wXes/UjS8n55qyQfS3JpklVJDuzXv7hftzrJe/p18/rR3dX9trf2638nyTeTXJTkrCSPmuxOSPKAJCcnuaT/2bdf/yd9v6uTvKVftzjJD5Icl+SyJKclWdhve0SSb/V9XJXkJ0m+BTyy3/7rUegkRyW5sO97WZL069+c5Pv9ef9r/8bmcOCtSVYm2T/JQ5Oc3rc5PclDJjinwWPtleTcvq4Lkmw92X0B7NTfb5cn+cuBc/5hko/3x/xCki2n6GPMu4HfSbISeC+wVb/vD5N8euCcf/2OPskfJLm4r/X0Cc7r0CTfSLKw3+89/Tn9KMn+fZt5Sd7b37+rkry2X//AJGf29+Pq/r6c8PmzISS5qf93aV/7RPfFuwce//f1637j04uxfsb1/euR7nSjKSf0x/hJkjdvqHNaD/PHP5/SXQ/+Jsl5SVYkeXySU5P83ySHw/qN4q6PJG/vXxNr9Xruvax//a1OsvcUxzg6ySeTfDvJj5Mc2q9fmuSMJJ8BLu3Xvax/3q9M8tEk8/r1h/Svhe8AT9ww98b0knwp3bX3siSH9ete3de2PN3189h+/f2TnNTffxcmGVndayvJoiRf669Xq5McPLBtYbpr6djjOO3rf0TnMKvP7f4+OaHf53tJnpdkC+CdwMH9c/bgidr1+z964Lm9KsnOs3rCVeXPBvqhC4U/Hrj9f4AjgeXAkn7dDsCafvlVwHED7f8f8KB++Y+Af5riWMuBp47bdzvgaOAvB9Z/AfgR3ajtSrpR12cC/wQcMtDui8BBA7eX9O0XAlsDPwaOAE4cawesAXYYaL+8X37PYO3AfelGh/8fcH+6TwS+TTcKvSfw7wNtt+v/PR3YuV/+PeDbU9wXnwPe0i/PA7bt+70UWARsBVwG7NE/Rr8Cdu/b/xvwsn75fOD5/b6rgfvRjXL/xwTnvv3A8T8JPLdfvgq4z7hzOZpuJH6s/VeAVw48zl8a327sWHQj8z8B9urXbwPMn+R+eBXwX33dC/tzWNKfcwFP7NudMFjPNM/n1f3yUuB64MF0b4bPA/YbeC4u6R/bnwEPG7yPxs4LeCNwysD9sxz4+375D4Fv9cuHAUf2y/ehG6F8GPC/gbcPPM5bM8nzZwO9vm+a6r4AtgcuBzLu8f/182ZcP+Pv368O3F/n9ue+A/ALYPMNdV7rcD9M+Hyiux68rl/3j8Cq/jG6P/Dz8ec8xHrHrgVbsvav5+X012jgSVPV3j9ul9C99nboXws79o/tzQOvi9+luwZs3t/+EN2AxQO55xq5BXAOcOyIHuOx1+7YdeRB/eO7PbA5cNZYbcBnuOda8BDgB6N+jq7FeR7Ib/4O3rY/z8XAt4BXDGyb8vU/ovpn/bkN/A33/E7cji4/LKL7/XLsQF+TtfsA8NJ+/RbAwtk8Z0diN7zbB5bvogtsv+KeUfAFU7S/e+D23Uw//aMmuX3zwLoAb6qq3fufh1XVaf36qewHfLmqbq2qG+kuujP1dOCDvy6q6lq6UeDlVXV1Vf0K+DTdC+cnwMOTfCDJHwA3JNmKbjrG59ONAn6U7gI/macCH+6PdVdVXd/Xf3JV3VxVN9GF9P379ldU1cp++SJgcbrRzQdV1cl9uy9W1S+q6ga64DXeU5Kcn+TS/viP7tevAj6d5GV0j/tE9qG78EN3UdlvinN7JPBfVXVhf3439PffZP69r/vW/pzH+v5ZVZ3TL39qmmNO5oKqurKq7qZ7g7N43PYnAGdW1RV9rYNTYV4OPAs4sKoGn/Nf7P+9aKC/ZwKv6B/78+lC+c7AhcAh6abMPKZ/Xt7r+bMO57UuJrovbgBuA45P8gLglvXo/2tVdXtVXQP8HHjA+hY8yyZ7Po29Vi4Fzq+qG6vqauC2JNsNu8je/nTXglvW4fUM8FmAqjoT2Gaa8xi7Zl4DnEH3CRt0z5cr+uWn0YWPC/vn+NOAh9O9WR+7Rt5B9+Z8VN6c5BLgu8BOdK/f71TVL6vqTuDzA22fDhzbn8spdPfRVJ8WbUwuBZ6e7hOh/fvfHQBfBj5WVZ+YZL/proXDsiGe288E/rx/PJfTZZZ7fVo4RbvzgL9I8n+Ah/a/i2aNIXY01tBdtGB254MdDJBkP7opA9dP0OZU4HVJNu/b7pJkEd283QOTbJbkAXTvLgfN5OORycJ5uHfAnrC/PuA+ju5F8Abg+L7P6waC9+5V9bszqGfa4/UmeqMxvv34+u/pOFlAN3pyUFU9BjiOe87/2XQBfk/goszsy3WTHouJ78u16aumWb82JrrfBk1V62q6C/2DJ+lzsL8J33j1F9on0c01/2SSV0zy/BmGe90X/ZuLvYGT6D5l+Ga//devk/6jvC3Wpf/1LXiWTfZ8GnwTPv4N+ijPYV1fzxPtO9VrZ6YDCx8feH4/sqqOnkHfQ5FkKV0w3aeqHgd8j+4Thsls1rcdO58H9W8wN3pV9SPuGc382yRjc0nPAZ41xTSBjen1OdvP7dANNow9ng+pqh9M1P1E7arqM8D/Am4FTk3y1PU4t3sxxI7G++iC5Ll0HzPNlmv7Pj9CN/d1IscD3wcuTjcX7aN0L7iTgCvpwsVH6Ua8BkPw2cBzkyzoR0afPUHfa7gnnA/O/T2N7qNjAJLct+//yUl26Od/vRj4Tro5tZtV1UnAO4DH9+8or0jywn7/JHncFPfD6cDr+rbzkmwDnAkckG6u3iK6aQJnTdZBf8wrkxzQ7/uCJNv3IwrPHdd87CJwTX/fjM092gzYqarOAP6M7iOWrYAb6T5WHXMu8KJ++aV09/VkfgjsmGSv/hhbTxOMn9HXvZAuSI2Nlj0kydi85xdPc8wx4+ueznl0j/HD+lq3H9j2PeC1wClJdpymnwnfeCV5KN3H0scB/wI8fqLnz1rUO6v658K2VfV14C10X4SA33ydPI/u49jWrcvzaVTOBJ6fbo7jjF/PA2YyWDDmef018350AwMXTtDmdOCgJL/V97t9/9w+H1ia5H79c/+FMz/FWbUtcG1V3ZLuuwhPoPu4+slJ7ttff6a63u9OI/pr0S1V9Sm639Nj14+j6KbxTPml1o3Ahnhunwq8aSzAJ9mjbzv+98GE7ZI8HPhJVb2fbmT4set/mvfY2N7Nb1Kqag3dF7XGbr9vYPPgA3lkv/1EurkrY+0XDyz/xrZJnFRVbxtXw9Hjbt8N/EX/8xuSHFFVN/UX3Avov3TQ73dhklPo5nj9lG5e4viL918B/5LkL+guwGP+GvhgH5rvAv6qqr6Y5G10H7EF+HpVfbkPpx/rAyDA2Pm8FPhwkiPpfun/a1/LRP4YWJbk1f3xXldV5yU5sT8vgOOr6nvpvmg1mZfTBfod6OZ+raCbY/Qb4beqrktyHN39tYZ7flHNAz6VZNv+HP+xb/sV4AvpJr6/CXgzcEKSPwWuBg6ZrKCquiPdlw0+0AfTW+lGSe715aDe2XRTFB4BfKa6v4ixGPgB8MokH6Wb3/zhKe6HsWP/Isk5/eN4K/A/07S/Ot2XQL7YP54/B54xsP3sdF8O/FqSZ0zWD90br8V0b7xCdx8dQBcK/jTJnXTn/wq6uXoTPX9GYWvgy/3oR4CxL5kd16+/gC7A3DzJ/i2Z6Pn0ptGWNLGqujjJ5+g+9v0pM389jxkbLNiGbg77VC4Avkb3seq7quqqJLuMO973++vaaf3z9k7gDVX13XRTZc6jm9t+Md01Zdi+CRyeZBXdCOx36T79+Bu66/xVdAMjY78P3kx3vV9FlzHOpPsyawseA7w3yd10j8Pr6L5HAt0b0ROS/F1V/dmoCpzKBnpuv4vuOzOr+uvvGuA5dL+7x6YP/O0U7Q6m+8LYncB/030hbNaMfeFAjUv3lwCOqO5P86xPH9vRfbz5d31wHty+VR9yt6S7MB1WVRevc9EaiT7EfrWqdpumqaR11AfQm8YNXmwyBn4fzAdOpvuTjCePui7NLY7ENibJB7n3n1r556paur59z6CPZen+iP0CujlcBlhJmpuOTvcH9BfQTSH40ojr0RzkSKyaleTt3Hue2Oer6phR1DMqSX6f7s+YDbqiqp4/inqkuSTJIXRTmAadU1VvGEU90lxiiJUkSVJz/OsEkiRJao4htmH9N7/nDM930zWXzhU8302d57vpmkvnChv/+Rpi27ZRP7k2AM930zWXzhU8302d57vpmkvnChv5+RpiJUmS1By/2DULdth+Xi3eafj/6c7Vv7iL+99v+H/7+vI1s/mfjM3cnXfczOZbLBr+gWfyH+5uAKM4381uvG2oxxtzR93GFlkwfcNNxMjOd4vR/Odgd9x1C1vM23L4B77rruEfE7jj7lvZYrOFQz/ubQ8ezeN71403M2/r4V6rFvz09ukbbQCjeu3+6r7Dfz4B/Oq2m5m/YPi/d2/5xZXXVNX9p2vn34mdBYt32pwLTt1p1GUMzVMOec2oSxiqmjeiFDsCC5dfNuoShmveKP4DpNHJg3971CUMVa67cdQlDNXlxzxw1CUMzc6HXj7qEobql8+Z6n9a3/Ss+Pj//ulM2jmdQJIkSc0xxEqSJKk5hlhJkiQ1xxArSZKk5hhiJUmS1BxDrCRJkppjiJUkSVJzDLGSJElqjiFWkiRJzTHESpIkqTmGWEmSJDXHECtJkqTmGGIlSZLUHEOsJEmSmmOIlSRJUnMMsZIkSWqOIVaSJEnNMcRKkiSpOYZYSZIkNccQK0mSpOYYYiVJktQcQ6wkSZKaY4iVJElScwyxkiRJao4hVpIkSc2ZkyE2yfxR1yBJkqR1N5QQm2Rxkh8kOS7JZUlOS7IwyfIkS/o2OyRZ0y+/KsmXknwlyRVJ3pjkT5J8L8l3k2w/xbGWJ/mnJOcmWZ1k73790UmWJTkN+ESSeUnem+TCJKuSvLZvt1mSD/V1fjXJ15MctOHvJUmSJM3UMEdidwY+WFWPBq4DDpym/W7AS4C9gWOAW6pqD+A84BXT7LuoqvYFXg+cMLB+T+B5VfUS4NXA9VW1F7AXcGiShwEvABYDjwFeA+wz4zOUJEnSUAzzY/Urqmplv3wRXVCcyhlVdSNwY5Lrga/06y8FHjvNvp8FqKozk2yTZLt+/SlVdWu//EzgsQOjrNvSBe39gM9X1d3Afyc5Y6IDJDkMOAzgIQ9ydoIkSdIwDXMk9vaB5bvoAvSvBmpYMEX7uwdu38304bsmuX3zwLoAb6qq3fufh1XVaf36aVXVsqpaUlVL7n+/eTPZRZIkSbNk1F/sWkP3ET/AbM47PRggyX50Uwaun6DNqcDrkmzet90lySLgbODAfm7sA4Cls1iXJEmSZsGoPwd/H/BvSV4OfHsW+702ybnANsAfTdLmeLopDRcnCXA1cABwEvA0YDXwI+B8YKIQLEmSpBEZSoitqjV0X9Qau/2+gc2D81uP7LefCJw40H7xwPJvbJvESVX1tnE1HD3u9t3AX/Q/vyHJEVV1U5L7ARfQzcOVJEnSRmLUI7Ebq6/2XwbbAnhXVf33qAuSJEnSPZoNsUk+CDxx3Op/rqql69v3bPQhSZKkDafZEFtVbxh1DZIkSRqNUf91AkmSJGmtGWIlSZLUHEOsJEmSmmOIlSRJUnMMsZIkSWqOIVaSJEnNMcRKkiSpOYZYSZIkNccQK0mSpOYYYiVJktQcQ6wkSZKaY4iVJElScwyxkiRJao4hVpIkSc0xxEqSJKk5hlhJkiQ1xxArSZKk5hhiJUmS1BxDrCRJkppjiJUkSVJzDLGSJElqjiFWkiRJzZk/6gI2BZev2YGnHPKaUZcxNGd87PhRlzBUT37tYaMuYWhq14ePuoShuvJp24y6hKFadFWNuoShWnjNdqMuYai2PXPu/Eqfa9eqq/e+e9QlDNfHZ9bMkVhJkiQ1xxArSZKk5hhiJUmS1BxDrCRJkppjiJUkSVJzDLGSJElqjiFWkiRJzTHESpIkqTmGWEmSJDXHECtJkqTmGGIlSZLUHEOsJEmSmmOIlSRJUnMMsZIkSWqOIVaSJEnNMcRKkiSpOYZYSZIkNccQK0mSpOYYYiVJktQcQ6wkSZKaY4iVJElScwyxkiRJao4hVpIkSc0xxEqSJKk5hlhJkiQ1xxArSZKk5hhiJUmS1Jx1DrFJTkxy0GwWI0mSJM2EI7GSJElqzoxCbJJ3JPlhkn9P8tkkR4zbvibJDv3ykiTL++WtknwsyaVJViU5sF//4n7d6iTv6dfN60d3V/fb3tqv/50k30xyUZKzkjxqijofkOTkJJf0P/v26/+k73d1krf06xYn+UGS45JcluS0JAv7bY9I8q2+j4uT/M4ExzosyYokK+684+aZ3I2SJEmaJfOna5BkCXAgsEff/mLgohn2/w7g+qp6TN/XfZPsCLwH2BO4FjgtyQHAz4AHVdVufdvt+j6WAYdX1Y+T/B7wIeCpkxzv/cB3qur5SeYBWyXZEzgE+D0gwPlJvtMfe2fgxVV1aJJ/68/zU8CngXdX1clJFjBB2K+qZX1tbL3tg2uG94ckSZJmwbQhFtgP+HJV3QqQ5Ctr0f/TgReN3aiqa5M8CVheVVf3/X0aeBLwLuDhST4AfI0u3G4F7At8PslYN/eZ4nhPBV7RH+su4Pok+wEnV9XN/fG+COwPnAJcUVUr+30vAhYn2ZouTJ/c93PbWpyvJEmShmAmITbTN+FX3DNauWDcvuNHKSfsrw+4jwN+H3gD8P8BbwGuq6rdZ1DDZKaq//aB5buAhdO0lyRJ0kZgJnNizwaem2RBPzL67AnarKGbHgDdR/JjTgPeOHYjyX2B84EnJ9mh/8j/xcB3+jm1m1XVSXTTEB5fVTcAVyR5Yb9/+qA7mdOB1/Vt5yXZBjgTOCDJlkkWAc8Hzpqsg/6YV/ZTHEhynyRbTnFMSZIkDdm0IbaqLqT76P0S4IvACuD6cc3+CvjnJGfRjWiO+Wvgvv0Xqi4BnlJV/wW8DTij7/Piqvoy8CBgeZKVwIl9G4CXAq/u978MeN4U5f4x8JQkl9JND3h0VV3c93cBXYA+vqq+N81pvxx4c5JVwLnAb0/TXpIkSUM0k+kEAO+rqqP7Eckzgb+vquPGNlbVWcAu43eqqpuAV06w/jPAZ8atuwR4/ARtrwD+YCZFVtX/MEHIrap/AP5h3Lo1wG4Dt983sPxjJv/ymCRJkkZspiF2WZJd6ea7frwf3ZQkSZJGYkYhtqpesqELWRtJ3g68cNzqz1fVMaOoR5IkScM105HYjUofVg2skiRJc5T/7awkSZKaY4iVJElScwyxkiRJao4hVpIkSc0xxEqSJKk5hlhJkiQ1xxArSZKk5hhiJUmS1BxDrCRJkppjiJUkSVJzDLGSJElqjiFWkiRJzTHESpIkqTmGWEmSJDXHECtJkqTmGGIlSZLUHEOsJEmSmmOIlSRJUnMMsZIkSWrO/FEXsEkI1LyMuoqhefJrDxt1CUP1nY8uG3UJQ7PPEYePuoSheuA5t4y6hKG65nFbjrqEodp+1U2jLmGobn7gdqMuYWiu32WrUZcwVDt/4uZRlzBUP51hO0diJUmS1BxDrCRJkppjiJUkSVJzDLGSJElqjiFWkiRJzTHESpIkqTmGWEmSJDXHECtJkqTmGGIlSZLUHEOsJEmSmmOIlSRJUnMMsZIkSWqOIVaSJEnNMcRKkiSpOYZYSZIkNccQK0mSpOYYYiVJktQcQ6wkSZKaY4iVJElScwyxkiRJao4hVpIkSc0xxEqSJKk5hlhJkiQ1xxArSZKk5hhiJUmS1BxDrCRJkppjiJUkSVJzDLGSJElqzvzZ7CzJ0cBNwDbAmVX1rbXcfylwR1WdO5PjVNX7krxzbY+V5FXAkqp649rUJ0mSpI3DrIbYMVV11DruupQuBE8ZYmfpWJIkSWrUek8nSPL2JJcn+RbwyH7diUkO6pePSnJhktVJliVJv/7NSb6fZFWSf02yGDgceGuSlUn2T/LQJKf3bU5P8pAJjj94rL2SnJvkkiQXJNl6itJ3SvLNvva/7PdfnOSHST7eH/MLSbac5LwPS7IiyYo777h53e9ASZIkrbX1CrFJ9gReBOwBvADYa4Jmx1bVXlW1G7AQeE6//s+BParqscDhVbUG+Ajwj1W1e1WdBRwLfKJv82ng/VPUsgXwOeCPq+pxwNOBW6cof2/gpcDuwAuTLOnXPxJY1h/zBuD1E+1cVcuqaklVLdl8i0VTHEaSJEmzbX1HYvcHTq6qW6rqBuCUCdo8Jcn5SS4Fngo8ul+/Cvh0kpcBv5qk/32Az/TLnwT2m6KWRwL/VVUXAlTVDVU1Wb8A/15Vv6iqW4EvDvT9s6o6p1/+1DTHlCRJ0gjMxl8nqMk2JFkAfAg4qKoeAxwHLOg3Pxv4ILAncFGSmczPnfRYQKbZPl1fNc16SZIkbSTWN8SeCTw/ycJ+/ulzx20fC6zXJNkKGJu7uhmwU1WdAfwZsB2wFXAjMDiP9Vy66QrQffR/9hS1/BDYMcle/TG2niYYPyPJ9kkWAgcAY6OvD0myT7/84mmOKUmSpBFYr79OUFUXJ/kcsBL4KXDWuO3XJTkOuBRYA1zYb5oHfCrJtnQjqP/Yt/0K8IUkzwPeBLwZOCHJnwJXA4dMUcsdSQ4GPtAH01vp5sXeNMkuZ9NNUXgE8JmqWtF/uewHwCuTfBT4MfDhtbhLJEmSNATr/Se2quoY4Jgpth8JHDnBpnvNNa2qHwGPHbf6qRO0O3pg+VUDyxcCT5hBzScCJ06y+e6qOny6PiRJkjQ6/o9dkiRJas4G+c8ONhZJfh94z7jVV1TV8ydq3/+Zr902dF2SJElaP5t0iK2qU4FTR12HJEmSZpfTCSRJktQcQ6wkSZKaY4iVJElScwyxkiRJao4hVpIkSc0xxEqSJKk5hlhJkiQ1xxArSZKk5hhiJUmS1BxDrCRJkppjiJUkSVJzDLGSJElqjiFWkiRJzTHESpIkqTmGWEmSJDXHECtJkqTmGGIlSZLUHEOsJEmSmmOIlSRJUnPmj7qATcFmN97GwuWXjbqMoaldHz7qEoZqnyMOH3UJQ3Pe+z4y6hKG6hGfnTuPLcC8W2vUJQzVDbtsM+oShmr7E84bdQlDc+pVK0ddwlA988BXjrqEjZIjsZIkSWqOIVaSJEnNMcRKkiSpOYZYSZIkNccQK0mSpOYYYiVJktQcQ6wkSZKaY4iVJElScwyxkiRJao4hVpIkSc0xxEqSJKk5hlhJkiQ1xxArSZKk5hhiJUmS1BxDrCRJkppjiJUkSVJzDLGSJElqjiFWkiRJzTHESpIkqTmGWEmSJDXHECtJkqTmGGIlSZLUHEOsJEmSmmOIlSRJUnMMsZIkSWqOIVaSJEnNGWqITbJdktf3y0uTfHWSdscn2XWKfo5OcsSGqlOSJEkbt2GPxG4HvH66RlX1mqr6/hDqkSRJUoOGHWLfDfxOkpXAe4GtknwhyQ+TfDpJAJIsT7KkX/6DJBcnuSTJ6eM7THJokm8kWdjv954kFyT5UZL9+zbzkrw3yYVJViV5bb/+gUnOTLIyyeok+/dtT+xvX5rkrUO7dyRJkjQj84d8vD8Hdquq3ZMsBb4MPBq4CjgHeCJw9ljjJPcHjgOeVFVXJNl+sLMkbwSeCRxQVbf3GXh+Ve2d5A+BvwSeDrwauL6q9kpyH+CcJKcBLwBOrapjkswDtgR2Bx5UVbv1x9huQ90ZkiRJWjfDDrHjXVBVVwL0o7OLGQixwBOAM6vqCoCq+uXAtpcDV9IF2DsH1n+x//eivj/ogu5jkxzU394W2Bm4EDghyebAl6pqZZKfAA9P8gHga8BpExWe5DDgMIAFWbSWpy1JkqT1Meq/TnD7wPJd3DtUB6hJ9l1NF1IfPEmfg/0FeFNV7d7/PKyqTquqM4EnAf8JfDLJK6rqWuBxwHLgDcDxEx28qpZV1ZKqWrJFFkxzmpIkSZpNww6xNwJbr0X784AnJ3kYwLjpBN8DXguckmTHafo5FXhdP+JKkl2SLEryUODnVXUc8C/A45PsAGxWVScB7wAevxb1SpIkaQiGOp2gqn6R5Jwkq4Fbgf+Zpv3V/cf2X0yyGfBz4BkD28/u/9TW15I8Y7J+6EZTFwMX918euxo4AFgK/GmSO4GbgFcADwI+1h8P4G1rf6aSJEnakIY+J7aqXjLJ+jcOLC8dWP4G8I1xbY8eWD6VbqSEVWHQAAAVQUlEQVQVulA6tv4a+jmxVXU38Bf9z6CP9z/jOfoqSZK0ERv1nFhJkiRprRliJUmS1BxDrCRJkppjiJUkSVJzDLGSJElqjiFWkiRJzTHESpIkqTmGWEmSJDXHECtJkqTmGGIlSZLUHEOsJEmSmmOIlSRJUnMMsZIkSWqOIVaSJEnNMcRKkiSpOYZYSZIkNccQK0mSpOYYYiVJktQcQ6wkSZKaY4iVJElScwyxkiRJao4hVpIkSc0xxEqSJKk5hlhJkiQ1Z/6oC9hkzJs36gqG5sqnbTPqEobqgefcMuoShuYRnz181CUM1X+8+COjLmGo9v2TufX4zjXzHvBboy5haObatWqnbe8adQkbJUdiJUmS1BxDrCRJkppjiJUkSVJzDLGSJElqjiFWkiRJzTHESpIkqTmGWEmSJDXHECtJkqTmGGIlSZLUHEOsJEmSmmOIlSRJUnMMsZIkSWqOIVaSJEnNMcRKkiSpOYZYSZIkNccQK0mSpOYYYiVJktQcQ6wkSZKaY4iVJElScwyxkiRJao4hVpIkSc0xxEqSJKk5hlhJkiQ1xxArSZKk5hhiJUmS1BxDrCRJkprTTIhNcu4s97c4yep+eUmS989m/5IkSdpw5o+6gJmqqn03YN8rgBUbqn9JkiTNrpZGYm/q/12aZHmSLyT5YZJPJ0m/7d1Jvp9kVZL39etOTHLQ+H7G9b00yVf75aOTnNAf4ydJ3jycM5QkSdJMNTMSO84ewKOBq4BzgCcm+T7wfOBRVVVJtluP/h8FPAXYGrg8yYer6s7BBkkOAw4DWJBF63EoSZIkra1mRmLHuaCqrqyqu4GVwGLgBuA24PgkLwBuWY/+v1ZVt1fVNcDPgQeMb1BVy6pqSVUt2SIL1uNQkiRJWluthtjbB5bvAuZX1a+AvYGTgAOAb/bbf0V/nv20gy3Wpf/1LViSJEmzp9UQey9JtgK2raqvA28Bdu83rQH27JefB2w+/OokSZI0mzalEcatgS8nWQAEeGu//rh+/QXA6cDNI6pPkiRJs6SZEFtVW/X/LgeWD6x/40CzvSfY73+AJwyselu/fg2w2/g+q+rocfvvtr61S5IkaXZtMtMJJEmSNHcYYiVJktQcQ6wkSZKaY4iVJElScwyxkiRJao4hVpIkSc0xxEqSJKk5hlhJkiQ1xxArSZKk5hhiJUmS1BxDrCRJkppjiJUkSVJzDLGSJElqjiFWkiRJzTHESpIkqTmGWEmSJDXHECtJkqTmGGIlSZLUHEOsJEmSmmOIlSRJUnMMsZIkSWqOIVaSJEnNMcRKkiSpOYZYSZIkNWf+qAvYJGyxOXnwb4+6iqFZdFWNuoShuuZxW466hKGZd+vcemz3/ZPDR13CUJ37Dx8ZdQlDtfQ1h466hKGqB2w/6hKGZt6tGXUJQ3XtLpuPuoTh+sbMmjkSK0mSpOYYYiVJktQcQ6wkSZKaY4iVJElScwyxkiRJao4hVpIkSc0xxEqSJKk5hlhJkiQ1xxArSZKk5hhiJUmS1BxDrCRJkppjiJUkSVJzDLGSJElqjiFWkiRJzTHESpIkqTmGWEmSJDXHECtJkqTmGGIlSZLUHEOsJEmSmmOIlSRJUnMMsZIkSWqOIVaSJEnNMcRKkiSpOYZYSZIkNccQK0mSpOYYYiVJktScTT7EJlmcZPU67rtjki/Mdk2SJElaP/NHXcDGrKquAg4adR2SJEn6TZv8SGxvfpKPJ1mV5AtJtkyyJsnfJDkvyYokj09yapL/m+RwWL9RXEmSJG04cyXEPhJYVlWPBW4AXt+v/1lV7QOcBZxIN+r6BOCd03WY5LA+/K64465bNkzVkiRJmtBcCbE/q6pz+uVPAfv1y6f0/14KnF9VN1bV1cBtSbabqsOqWlZVS6pqyRbzttwwVUuSJGlCcyXE1iS3b+//vXtgeey284UlSZI2UnMlxD4kyT798ouBs0dZjCRJktbPXAmxPwBemWQVsD3w4RHXI0mSpPWwyX9kXlVrgF0n2LR4oM2JdF/sGrs9tu0aYLcNVZskSZLWzVwZiZUkSdImxBArSZKk5hhiJUmS1BxDrCRJkppjiJUkSVJzDLGSJElqjiFWkiRJzTHESpIkqTmGWEmSJDXHECtJkqTmGGIlSZLUHEOsJEmSmmOIlSRJUnMMsZIkSWqOIVaSJEnNMcRKkiSpOYZYSZIkNccQK0mSpOYYYiVJktQcQ6wkSZKaY4iVJElScwyxkiRJao4hVpIkSc0xxEqSJKk580ddwCbhrrvIdTeOuoqhWXjNdqMuYai2X3XTqEsYmht22WbUJWgDWvqaQ0ddwlAtP/64UZcwVM965otGXcLQ7HDJ3aMuYagyt053xhyJlSRJUnMMsZIkSWqOIVaSJEnNMcRKkiSpOYZYSZIkNccQK0mSpOYYYiVJktQcQ6wkSZKaY4iVJElScwyxkiRJao4hVpIkSc0xxEqSJKk5hlhJkiQ1xxArSZKk5hhiJUmS1BxDrCRJkppjiJUkSVJzDLGSJElqjiFWkiRJzTHESpIkqTmGWEmSJDXHECtJkqTmGGIlSZLUHEOsJEmSmmOIlSRJUnMMsZIkSWqOIVaSJEnNGUmITXJ0kiOSvDPJ06dpuzzJkmHVNu7Yr0py7CiOLUmSpMnNH+XBq+qoYR8zybyqumvYx5UkSdLsGdpIbJK3J7k8ybeAR/brTkxyUL98VJILk6xOsixJBnZ/WZJz+217T3GMo5N8Msm3k/w4yaH9+qVJzkjyGeDSft3LklyQZGWSjyaZ168/JMmPknwHeOIUxzosyYokK+64+9b1vXskSZK0FoYSYpPsCbwI2AN4AbDXBM2Oraq9qmo3YCHwnIFti6pqX+D1wAnTHO6xwLOBfYCjkuzYr98beHtV7Zrkd4GDgSdW1e7AXcBLkzwQ+Cu68PoMYNfJDlJVy6pqSVUt2WKzhdOUJEmSpNk0rOkE+wMnV9UtAElOmaDNU5L8GbAlsD1wGfCVfttnAarqzCTbJNmuqq6b5FhfrqpbgVuTnEEXXq8DLqiqK/o2TwP2BC7sB3wXAj8Hfg9YXlVX93V+DthlPc5bkiRJG8Aw58TWZBuSLAA+BCypqp8lORpYMMW+k/Y1RdubBw8JfLyq3jaujgOm6VuSJEkbgWHNiT0TeH6ShUm2Bp47bvtYYL0myVbAQeO2HwyQZD/g+qq6fopjPS/JgiT3A5YCF07Q5nTgoCS/1fe7fZKHAucDS5PcL8nmwAtnfoqSJEkalqGMxFbVxf1H8yuBnwJnjdt+XZLj6L50tYZ7B89rk5wLbAP80TSHuwD4GvAQ4F1VdVWS35gSUFXfT3IkcFqSzYA7gTdU1Xf7UeDzgP8CLgbmre35SpIkacMa2nSCqjoGOGaK7UcCR06wfulaHupHVXXYuD6WA8vHrfsc8LkJjvcx4GNreUxJkiQNkf9jlyRJkpoz0v/sYF0lOQT443Grz6mqN4yiHkmSJA1XkyHWj/wlSZLmNqcTSJIkqTmGWEmSJDXHECtJkqTmGGIlSZLUHEOsJEmSmmOIlSRJUnMMsZIkSWqOIVaSJEnNMcRKkiSpOYZYSZIkNccQK0mSpOYYYiVJktQcQ6wkSZKaY4iVJElScwyxkiRJao4hVpIkSc0xxEqSJKk5hlhJkiQ1xxArSZKk5swfdQGbgtsevDmXH/PAUZcxNNueObeeNjc/cLtRlzA0259w3qhLGKp5D/itUZcwVPWA7UddwlA965kvGnUJQ/WN0/511CUMze/vuPuoSxiqXx6yz6hL2Cg5EitJkqTmGGIlSZLUHEOsJEmSmmOIlSRJUnMMsZIkSWqOIVaSJEnNMcRKkiSpOYZYSZIkNccQK0mSpOYYYiVJktQcQ6wkSZKaY4iVJElScwyxkiRJao4hVpIkSc0xxEqSJKk5hlhJkiQ1xxArSZKk5hhiJUmS1BxDrCRJkppjiJUkSVJzDLGSJElqjiFWkiRJzTHESpIkqTmGWEmSJDXHECtJkqTmGGIlSZLUHEOsJEmSmmOIlSRJUnMMsZIkSWrOnAixSb6U5KIklyU5rF/36iQ/SrI8yXFJju3X3z/JSUku7H+eONrqJUmSNN78URcwJH9UVb9MshC4MMnXgHcAjwduBL4NXNK3/WfgH6vq7CQPAU4Ffnd8h30YPgxg3g7bDuEUJEmSNGauhNg3J3l+v7wT8HLgO1X1S4Aknwd26bc/Hdg1ydi+2yTZuqpuHOywqpYBywDu8/AH1QauX5IkSQM2+RCbZCldMN2nqm5Jshy4nAlGV3ub9W1vHU6FkiRJWltzYU7stsC1fYB9FPAEYEvgyUnum2Q+cOBA+9OAN47dSLL7UKuVJEnStOZCiP0mMD/JKuBdwHeB/wT+Bjgf+BbwfeD6vv2bgSVJViX5PnD48EuWJEnSVDb56QRVdTvwrPHrk6yoqmX9SOzJdCOwVNU1wMHDrVKSJElrYy6MxE7m6CQrgdXAFcCXRlyPJEmSZmiTH4mdTFUdMeoaJEmStG7m8kisJEmSGmWIlSRJUnMMsZIkSWqOIVaSJEnNMcRKkiSpOYZYSZIkNccQK0mSpOYYYiVJktQcQ6wkSZKaY4iVJElScwyxkiRJao4hVpIkSc0xxEqSJKk5hlhJkiQ1xxArSZKk5hhiJUmS1BxDrCRJkppjiJUkSVJzDLGSJElqjiFWkiRJzTHESpIkqTnzR13ApmDBT29n50MvH3UZQ1O7PnzUJQzV9btsNeoShubUq1aOuoShesRnDx91CUM179aMuoSh2uGSu0ddwlD9/o67j7qEoZlr16pnHvi4UZewUXIkVpIkSc0xxEqSJKk5hlhJkiQ1xxArSZKk5hhiJUmS1BxDrCRJkppjiJUkSVJzDLGSJElqjiFWkiRJzTHESpIkqTmGWEmSJDXHECtJkqTmGGIlSZLUHEOsJEmSmmOIlSRJUnMMsZIkSWqOIVaSJEnNMcRKkiSpOYZYSZIkNccQK0mSpOYYYiVJktQcQ6wkSZKaY4iVJElScwyxkiRJao4hVpIkSc0xxEqSJKk5cyrEJlmTZIcJ1p87inokSZK0buZUiJ1MVe076hokSZI0c5tsiE2yKMnXklySZHWSgwe2LUzyzSSH9rdv6v9dmmR5ki8k+WGSTyfJqM5BkiRJE9tkQyzwB8BVVfW4qtoN+Ga/fivgK8Bnquq4CfbbA3gLsCvwcOCJwyhWkiRJM7cph9hLgacneU+S/avq+n79l4GPVdUnJtnvgqq6sqruBlYCiydqlOSwJCuSrLijbpv14iVJkjS5TTbEVtWPgD3pwuzfJjmq33QO8KwppgncPrB8FzB/kv6XVdWSqlqyRRbMVtmSJEmagU02xCbZEbilqj4FvA94fL/pKOAXwIdGVZskSZLWzyYbYoHHABckWQm8HfjrgW1vARYk+buRVCZJkqT1MuFH5ZuCqjoVOHXc6sUDy4cMtN2q/3c5sHxg/Rs3WIGSJElaZ5vySKwkSZI2UYZYSZIkNccQK0mSpOYYYiVJktQcQ6wkSZKaY4iVJElScwyxkiRJao4hVpIkSc0xxEqSJKk5hlhJkiQ1xxArSZKk5hhiJUmS1BxDrCRJkppjiJUkSVJzDLGSJElqjiFWkiRJzTHESpIkqTmGWEmSJDXHECtJkqTmGGIlSZLUHEOsJEmSmmOIlSRJUnMMsZIkSWqOIVaSJEnNSVWNuobmLdphp9r1OW8ddRlDc/Xed4+6hKHa+RO3jLqEoal5c+t97Z3bbjHqEobq2l02H3UJQ7XVf9416hKG6vZt5s7r974/nDvXZYDTTvr4qEsYqnkP/I+LqmrJdO3mzjNekiRJmwxDrCRJkppjiJUkSVJzDLGSJElqjiFWkiRJzTHESpIkqTmGWEmSJDXHECtJkqTmGGIlSZLUHEOsJEmSmmOIlSRJUnMMsZIkSWqOIVaSJEnNMcRKkiSpOYZYSZIkNccQK0mSpOYYYiVJktQcQ6wkSZKaY4iVJElScwyxkiRJao4hVpIkSc0xxEqSJKk5hlhJkiQ1xxArSZKk5hhiJUmS1BxDrCRJkpozf9QFjJfkaOAmYBvgzKr61hRtlwNHVNWKGfa9O7BjVX19FkqVJEnSiGx0IXZMVR21AbrdHVgCGGIlSZIatlFMJ0jy9iSXJ/kW8Mh+3YlJDuqXj0pyYZLVSZYlycDuL0tybr9t7779oiQn9Pt8L8nzkmwBvBM4OMnKJAdP1K7f/9FJLujbrUqy83DvEUmSJE1l5CE2yZ7Ai4A9gBcAe03Q7Niq2quqdgMWAs8Z2LaoqvYFXg+c0K97O/DtqtoLeArwXmBz4Cjgc1W1e1V9bqJ2SRYBhwP/XFVjI7dXTlD3YUlWJFnxq9tuXs97QZIkSWtjY5hOsD9wclXdApDklAnaPCXJnwFbAtsDlwFf6bd9FqCqzkyyTZLtgGcC/yvJEX2bBcBDJuh3snbnAW9P8mDgi1X14/E7VtUyYBnAoh12qrU8Z0mSJK2HjSHEAkwaApMsAD4ELKmqn/Vf/Fowxb4FBDiwqi4f19fvje9+onbAD5KcDzwbODXJa6rq2zM+G0mSJG1QI59OAJwJPD/JwiRbA88dt30ssF6TZCvgoHHbDwZIsh9wfVVdD5wKvGls7mySPfq2NwJbD+w7YbskDwd+UlXvB04BHrv+pylJkqTZMvIQW1UXA58DVgInAWeN234dcBxwKfAl4MJxXVyb5FzgI8Cr+3XvopsDuyrJ6v42wBnArmNf7Jqi3cHA6iQrgUcBn5il05UkSdIs2CimE1TVMcAxU2w/EjhygvVLJ2l/K/DaCdb/knt/cWyidn8L/O2URUuSJGlkRj4SK0mSJK0tQ6wkSZKaY4iVJElScwyxkiRJao4hVpIkSc0xxEqSJKk5hlhJkiQ1xxArSZKk5hhiJUmS1BxDrCRJkppjiJUkSVJzDLGSJElqjiFWkiRJzTHESpIkqTmGWEmSJDXHECtJkqTmGGIlSZLUHEOsJEmSmmOIlSRJUnMMsZIkSWqOIVaSJEnNMcRKkiSpOYZYSZIkNccQK0mSpOakqkZdQ/OSXA38dASH3gG4ZgTHHRXPd9M1l84VPN9Nnee76ZpL5wqjO9+HVtX9//927dgKQBAIouDRk9Vakz1hCWbgHjMxAT/bx+PrkBEbbIzxzDmv3fdYRW9fJ7VW6e1Ob18ntVb9v9d3AgAA4hixAADEMWKz3bsvsJjevk5qrdLbnd6+Tmqt+nmvP7EAAMTxEgsAQBwjFgCAOEYsAABxjFgAAOIYsQAAxHkBHhjiXJyPNCcAAAAASUVORK5CYII=\n", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "check(df)" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "del df['skin']" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Check Data Types" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [], + "source": [ + "diabetes_map = {True:1, False:0}" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [], + "source": [ + "df['diabetes'] = df['diabetes'].map(diabetes_map)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
num_pregglucose_concdiastolic_bpthicknessinsulinbmidiab_predagediabetes
061487235033.60.627501
11856629026.60.351310
28183640023.30.672321
318966239428.10.167210
40137403516843.12.288331
\n", + "
" + ], + "text/plain": [ + " num_preg glucose_conc diastolic_bp thickness insulin bmi diab_pred \\\n", + "0 6 148 72 35 0 33.6 0.627 \n", + "1 1 85 66 29 0 26.6 0.351 \n", + "2 8 183 64 0 0 23.3 0.672 \n", + "3 1 89 66 23 94 28.1 0.167 \n", + "4 0 137 40 35 168 43.1 2.288 \n", + "\n", + " age diabetes \n", + "0 50 1 \n", + "1 31 0 \n", + "2 32 1 \n", + "3 21 0 \n", + "4 33 1 " + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df.head(5)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Spliting the data\n", + "70% for training, 30% for testing" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "C:\\Users\\ronglu\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\cross_validation.py:41: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. Also note that the interface of the new CV iterators are different from that of this module. This module will be removed in 0.20.\n", + " \"This module will be removed in 0.20.\", DeprecationWarning)\n" + ] + } + ], + "source": [ + "from sklearn.cross_validation import train_test_split\n", + "\n", + "feature_col_names = ['num_preg', 'glucose_conc', 'diastolic_bp', 'thickness', 'insulin', 'bmi', 'diab_pred', 'age']\n", + "predicted_class_names = ['diabetes']\n", + "\n", + "x = df[feature_col_names].values # predictor feature columns (8 X m)\n", + "y = df[predicted_class_names].values # predicted class (1=true, 0=false) column (1 X m)\n", + "split_test_size = 0.30\n", + "\n", + "x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=split_test_size, random_state=42)\n", + "# test_size = 0.3 is 30%, 42 is the answer to everything" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We check to ensure we have the desired 70% train, 30% test split of the data" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "69.92% in training set\n", + "30.08% in test set\n" + ] + } + ], + "source": [ + "print(\"{0:0.2f}% in training set\".format((len(x_train)/len(df.index))*100))\n", + "print(\"{0:0.2f}% in test set\".format((len(x_test)/len(df.index))*100))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Post-split Data Preparation" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Impute with the mean" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [], + "source": [ + "from sklearn.preprocessing import Imputer\n", + "\n", + "# Impute with mean all 0 readings\n", + "fill_0 = Imputer(missing_values=0, strategy=\"mean\", axis=0)\n", + "\n", + "x_train = fill_0.fit_transform(x_train)\n", + "x_test = fill_0.fit_transform(x_test)\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Training Initial Algorithm = Naive Bayes" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "GaussianNB(priors=None)" + ] + }, + "execution_count": 13, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from sklearn.naive_bayes import GaussianNB\n", + "\n", + "# create Gaussian Naive Bayes model object and train it with the data\n", + "nb_model = GaussianNB()\n", + "\n", + "nb_model.fit(x_train, y_train.ravel())" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Performance on Training data" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Accurary: 0.7542\n", + "\n" + ] + } + ], + "source": [ + "# predict values using the training data\n", + "nb_predict_train = nb_model.predict(x_train)\n", + "\n", + "# import the performance metrics library\n", + "from sklearn import metrics\n", + "\n", + "# Accuracy\n", + "print(\"Accurary: {0:.4f}\".format(metrics.accuracy_score(y_train, nb_predict_train)))\n", + "print(\"\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Performance on Testing data" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Accurary: 0.7359\n", + "\n" + ] + } + ], + "source": [ + "# predict values using the training data\n", + "nb_predict_test = nb_model.predict(x_test)\n", + "\n", + "# import the performance metrics library\n", + "# from sklearn import metrics\n", + "\n", + "# Accuracy\n", + "print(\"Accurary: {0:.4f}\".format(metrics.accuracy_score(y_test, nb_predict_test)))\n", + "print(\"\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Metrics" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Confusion Matrix\n", + "[[118 33]\n", + " [ 28 52]]\n", + "\n", + "Classification Report\n", + " precision recall f1-score support\n", + "\n", + " 0 0.81 0.78 0.79 151\n", + " 1 0.61 0.65 0.63 80\n", + "\n", + "avg / total 0.74 0.74 0.74 231\n", + "\n" + ] + } + ], + "source": [ + "print(\"Confusion Matrix\")\n", + "print(\"{0}\".format(metrics.confusion_matrix(y_test, nb_predict_test)))\n", + "print(\"\")\n", + "\n", + "print(\"Classification Report\")\n", + "print(metrics.classification_report(y_test, nb_predict_test))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Retrain = Random Forest " + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "RandomForestClassifier(bootstrap=True, class_weight=None, criterion='gini',\n", + " max_depth=None, max_features='auto', max_leaf_nodes=None,\n", + " min_impurity_decrease=0.0, min_impurity_split=None,\n", + " min_samples_leaf=1, min_samples_split=2,\n", + " min_weight_fraction_leaf=0.0, n_estimators=10, n_jobs=1,\n", + " oob_score=False, random_state=42, verbose=0, warm_start=False)" + ] + }, + "execution_count": 17, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from sklearn.ensemble import RandomForestClassifier\n", + "\n", + "rf_model = RandomForestClassifier(random_state=42) # Create random forest object\n", + "\n", + "rf_model.fit(x_train, y_train.ravel())" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Performance on Training data" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Accurary: 0.9870\n", + "\n" + ] + } + ], + "source": [ + "# predict values using the training data\n", + "rf_predict_train = rf_model.predict(x_train)\n", + "\n", + "# import the performance metrics library\n", + "from sklearn import metrics\n", + "\n", + "# Accuracy\n", + "print(\"Accurary: {0:.4f}\".format(metrics.accuracy_score(y_train, rf_predict_train)))\n", + "print(\"\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Performance on Testing data" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Accurary: 0.7100\n", + "\n" + ] + } + ], + "source": [ + "# predict values using the testing data\n", + "rf_predict_test = rf_model.predict(x_test)\n", + "\n", + "# import the performance metrics library\n", + "from sklearn import metrics\n", + "\n", + "# Accuracy\n", + "print(\"Accurary: {0:.4f}\".format(metrics.accuracy_score(y_test, rf_predict_test)))\n", + "print(\"\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Metrics" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Confusion Matrix\n", + "[[121 30]\n", + " [ 37 43]]\n", + "\n", + "Classification Report\n", + " precision recall f1-score support\n", + "\n", + " 0 0.77 0.80 0.78 151\n", + " 1 0.59 0.54 0.56 80\n", + "\n", + "avg / total 0.70 0.71 0.71 231\n", + "\n" + ] + } + ], + "source": [ + "print(\"Confusion Matrix\")\n", + "print(\"{0}\".format(metrics.confusion_matrix(y_test, rf_predict_test)))\n", + "print(\"\")\n", + "\n", + "print(\"Classification Report\")\n", + "print(metrics.classification_report(y_test, rf_predict_test))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Retrain = Logistic Regression" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "LogisticRegression(C=0.7, class_weight='balanced', dual=False,\n", + " fit_intercept=True, intercept_scaling=1, max_iter=100,\n", + " multi_class='ovr', n_jobs=1, penalty='l2', random_state=42,\n", + " solver='liblinear', tol=0.0001, verbose=0, warm_start=False)" + ] + }, + "execution_count": 21, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from sklearn.linear_model import LogisticRegression\n", + "\n", + "lf_model = LogisticRegression(C=0.7, class_weight=\"balanced\", random_state=42)\n", + "lf_model.fit(x_train, y_train.ravel())" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Performance on Training data" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Accurary: 0.7765\n", + "\n" + ] + } + ], + "source": [ + "# predict values using the training data\n", + "lf_predict_train = lf_model.predict(x_train)\n", + "\n", + "# import the performance metrics library\n", + "from sklearn import metrics\n", + "\n", + "# Accuracy\n", + "print(\"Accurary: {0:.4f}\".format(metrics.accuracy_score(y_train, lf_predict_train)))\n", + "print(\"\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Performance on Testing data" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Accurary: 0.7056\n", + "\n" + ] + } + ], + "source": [ + "# predict values using the training data\n", + "lf_predict_test = lf_model.predict(x_test)\n", + "\n", + "# import the performance metrics library\n", + "from sklearn import metrics\n", + "\n", + "# Accuracy\n", + "print(\"Accurary: {0:.4f}\".format(metrics.accuracy_score(y_test, lf_predict_test)))\n", + "print(\"\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Metrics" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Confusion Matrix\n", + "[[105 46]\n", + " [ 22 58]]\n", + "\n", + "Classification Report\n", + " precision recall f1-score support\n", + "\n", + " 0 0.83 0.70 0.76 151\n", + " 1 0.56 0.72 0.63 80\n", + "\n", + "avg / total 0.73 0.71 0.71 231\n", + "\n" + ] + }, + { + "data": { + "text/plain": [ + "[,\n", + " ]" + ] + }, + "execution_count": 31, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAD8CAYAAAB5Pm/hAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3Xd4VGXax/HvnU5Ch9CbdGkKRumJha6CIrpWUFGxgrBr211dd/V1Le+GYkOwIXYBARWkqQkgxSDSe+8JRXoLed4/Mr6yGEhIOzOT3+e6uGbm5MzM/ZDw48lzztzHnHOIiEjwCvG6ABERKVgKehGRIKegFxEJcgp6EZEgp6AXEQlyCnoRkSCnoBcRCXIKehGRIKegFxEJcmFeFwBQvnx5V6tWLa/LEBEJKAsWLNjtnIvNbj+/CPpatWqRkpLidRkiIgHFzDblZD8t3YiIBDkFvYhIkFPQi4gEOQW9iEiQU9CLiAQ5Bb2ISJBT0IuIBLmADvp1aYf4z9RVHDt5yutSRET8VkAH/bTlu3j1u7VcPWwmCzbt9bocERG/FNBBf39CHUbdfRnHTmbQa/gcnp24jMPH070uS0TErwR00AMk1I9lysB4ereqyag5G+k0OJnk1WlelyUi4jcCPugBikeG8c8eTfi8X2siw0Po/e58/vLFIn49csLr0kREPBcUQf+bS2uVZVL/9jx4eR2+XLiNDonJTF6yw+uyREQ8FVRBDxAVHsrjXRoy4aG2VCgRyQMf/cwDHy4g9eAxr0sTEfFE0AX9b5pULcWEh9vyeJcGzFiZSsfEZL5I2YJzzuvSREQKVdAGPUB4aAgPXl6XyQPaU79icR4bs5je785ny94jXpcmIlJosg16M3vXzFLNbOlp28qa2TQzW+O7LePbbmY2zMzWmtliM2tRkMXnVJ3Y4nx2X2v+1aMxP2/aR+chybw/ewMZGZrdi0jwy8mM/n2gyxnbngRmOOfqATN8jwG6AvV8f+4D3syfMvMuJMTo3boWUwbGE1erLM9+tZwb35rD2tSDXpcmIlKgsg1651wycObHTnsAo3z3RwHXnbb9A5dpLlDazCrnV7H5oVqZaEbddSn/ufEi1qYeotvQWbz+/VpOnsrwujQRkQKR2zX6is65HQC+2wq+7VWBLaftt9W3za+YGTdcUo3pgxLo0KgCr0xZRY/XZrN0236vSxMRyXf5fTDWstiW5UK4md1nZilmlpKW5s0nWWNLRPLGbZcw/PZLSDt0nB6vz+alb1eqSZqIBJXcBv2u35ZkfLepvu1bgeqn7VcN2J7VCzjnRjjn4pxzcbGxsbksI390aVKJ6QMTuKFFVd78YR3dhs5k/gY1SROR4JDboJ8I9PHd7wNMOG17b9/ZN62A/b8t8fi7UtHhvNzrIj7s25ITpzK46a05PD1+KYfUJE1EAlxOTq/8BJgDNDCzrWbWF3gR6Ghma4COvscAk4D1wFpgJPBggVRdgNrVK8+UR+O5q20tPpy3iU6JSXy/KjX7J4qI+Cnzh0+KxsXFuZSUFK/L+IMFm/bxxNjFrE09RM/mVXn6mkaUiYnwuiwREQDMbIFzLi67/YL6k7F5dUnNMnzTvx2PXFmXiYu203FwEt8s3qE2CiISUBT02YgMC+XPnRow8eF2VC5VjIc+/pl+oxeQekBN0kQkMCjoc6hRlZJ8+WAbnurakKTVaVyVmMTnP6lJmoj4PwX9eQgLDaFfQh0mD2jPhZVL8vjYxdz+zjw271GTNBHxXwr6XKgdW5xP723F89c1YdGW/XQeksw7szZwSk3SRMQPKehzKSTEuL1VTaYOjKdl7bI89/Vyeg3/kTW71CRNRPyLgj6PqpQuxnt3XsqQP13Mxt2HuXrYLIbNWMOJdDVJExH/oKDPB2bGdc2rMm1QAp2bVCJx2mq6vzaLxVt/9bo0EREFfX4qXzySV29pzsjecew7coLrXp/Nvyet4OgJNUkTEe8o6AtAx0YVmTowgT9dWp23ktfTdWgyc9fv8bosESmiFPQFpFSxcP7dsxkf39OSDAc3j5jL375cwsFjJ70uTUSKGAV9AWtTtzzfPtqee9pdwCfzN9NpcDLfrdzldVkiUoQo6AtBdEQYf7+mEWMfaEOJqDDufj+FRz9dyN7DJ7wuTUSKAAV9IWpeowxfP9KeAVfV45slO+iQmMTERdvVRkFECpSCvpBFhIUwsGN9vnqkHdXLFKP/Jwu594MF7NyvJmkiUjAU9B5pWKkk4x5sy9+6XcistWl0TEzik/mbNbsXkXynoPdQaIhxb3xtvh0QT+OqJXlq3BJuHTmPTXsOe12aiAQRBb0fqFU+ho/vacW/ezZl6bbMJmlvz1yvJmkiki8U9H4iJMS45bIaTBuUQLu65Xn+mxX0fPNHVu1UkzQRyRsFvZ+pVCqKkb3jGHZLc7bsPcI1r85k8LTVapImIrmmoPdDZkb3i6owfVAC3ZpWZuiMNVzz6kx+2aImaSJy/hT0fqxsTARDb27OO33iOHA0nZ5vzOb5r5erSZqInBcFfQC46sKKTB0Uz82X1eDtWRvoPCSZH9ft9rosEQkQCvoAUTIqnBeub8on97YixODWkfN4atxiDqhJmohkQ0EfYFrXKcfkAfH0i6/NZz9toWNiEtOXq0maiJydgj4AFYsI5aluFzL+obaUiY7gng9SeOSThew+dNzr0kTEDynoA1izaqWZ+HA7BnWsz7dLd9AxMYnxC7epjYKI/BcFfYCLCAuh/1X1+KZ/e2qWi+HRz36h76gUtv961OvSRMRPKOiDRP2KJRj7QBuevqYRc9btodPgZD6cu4kMtVEQKfIU9EEkNMTo2+4Cpjwaz0XVS/H38Uu5ZeRcNuxWkzSRokxBH4RqlIvmw74tefmGZizfcYAuQ5J5K2kd6afURkGkKFLQBykz46ZLqzN9UALx9WP59+SVXP/GjyzffsDr0kSkkCnog1zFklGMuOMSXr+1BTv2H6X7a7P4z9RVHE9XGwWRokJBXwSYGVc3q8y0gQl0v6gKr363lquHzWLBpn1elyYihUBBX4SUiYkg8U8X895dl3LkeDq9hv/IP79axpET6V6XJiIFSEFfBF3RoAJTByVwR6uavDd7I50GJzNrjZqkiQQrBX0RVTwyjH/1aMLn/VoTHhrC7e/M4/Exi9h/VE3SRIKNgr6Iu+yCskwe0J4HLq/D2J+30TExiSnLdnpdlojkIwW9EBUeyhNdGjL+wbaUKx5Jv9ELeOijn0k7qCZpIsEgT0FvZgPNbJmZLTWzT8wsyswuMLN5ZrbGzD4zs4j8KlYKVtNqpZj4cFse69yAact30SExibELtqpJmkiAy3XQm1lVoD8Q55xrAoQCNwMvAYOdc/WAfUDf/ChUCkd4aAgPXVGXSQPaUbdCcf78xSLufO8ntqlJmkjAyuvSTRhQzMzCgGhgB3AlMMb39VHAdXl8D/FA3Qol+KJfa569thE/bdxLp8QkPpizUU3SRAJQroPeObcN+F9gM5kBvx9YAPzqnPvtxOytQNW8FineCAkx7myb2SStRc0yPDNhGX8aMYd1aYe8Lk1EzkNelm7KAD2AC4AqQAzQNYtds5wCmtl9ZpZiZilpaWm5LUMKQfWy0Xxw92W80qsZq3YepOvQmbzxw1pOqkmaSEDIy9JNB2CDcy7NOXcSGAe0AUr7lnIAqgHbs3qyc26Ecy7OORcXGxubhzKkMJgZN8ZVZ/qfE7iyQQVe/nYV170+m6Xb9ntdmohkIy9BvxloZWbRZmbAVcBy4Hugl2+fPsCEvJUo/qRCiSiG33EJb97Wgl0HjtPj9dm8MmUlx06qSZqIv8rLGv08Mg+6/gws8b3WCOAJYJCZrQXKAe/kQ53iZ7o2rcz0QfFc37wqr3+/jm7DZpKyca/XZYlIFswfzpGOi4tzKSkpXpchuZS8Oo2nxi1h+/6j9Gldi8c6NyAmMiz7J4pInpjZAudcXHb76ZOxkmfx9WOZOjCePq1rMWpOZpO0pNU6wC7iLxT0ki9iIsN4tntjvujXmsjwEPq8O58/f76IX4+c8Lo0kSJPQS/5Kq5WWSb1b89DV9Rh/C/b6JCYzOQlO7wuS6RIU9BLvosKD+Wxzg2Z+HBbKpaM5IGPfub+0QtIPXDM69JEiiQFvRSYxlVKMeGhtjzRpSHfrUqlQ2ISX6RsUZM0kUKmoJcCFRYawgOX12HygPY0qFSCx8Yspve789my94jXpYkUGQp6KRR1Yovz2X2tea5HY37etI/OQ5J5f/YGTqlJmkiBU9BLoQkJMe5oXYspA+O5tFZZnv1qOTe9NYe1qQe9Lk0kqCnopdBVKxPN+3ddSuJNF7Eu7RDdhs7ite/WqEmaSAFR0IsnzIyeLaoxbWACHRtX5H+nrqb7a2qSJlIQFPTiqdgSkbx+awveuuMSdh/KbJL24mQ1SRPJTwp68QudG1di+sAEerWoxvCkdXQbOpP5G9QkTSQ/KOjFb5SKDuelXs34sG9LTpzK4Ka35vD0+KUcPHbS69JEApqCXvxOu3rlmTownrvbXsCH8zbReXAy369K9boskYCloBe/FB0RxjPXNmLM/W2IiQzjrvd+YtBnv7DvsJqkiZwvBb34tUtqluHr/u3of2VdJi7aTofEJL5evF1tFETOg4Je/F5kWCiDOjXgq0faUaV0MR7+eCH9Ri9gl5qkieSIgl4CxoWVS/Llg214qmtDklan0SExic9+2qzZvUg2FPQSUMJCQ+iXUIdvH43nwsoleWLsEm57ex6b96hJmsjZKOglIF1QPoZP723F/1zfhMVb99N5SDLvzFKTNJGsKOglYIWEGLe1rMm0QfG0rlOO575ezg1v/sjqXWqSJnI6Bb0EvMqlivFOnziG3nwxm/Yc5uphMxk2Yw0n0tUkTQQU9BIkzIweF1dl+qAEujSpTOK01XR/bRaLtvzqdWkinlPQS1ApVzySV29pzsjecew7coLr35jNC5NWcPSEmqRJ0aWgl6DUsVFFpg1K4E+XVmdE8nq6Dk1mzro9Xpcl4gkFvQStklHh/LtnMz6+pyUZDm4ZOZe/frmEA2qSJkWMgl6CXpu65ZnyaDz3tr+AT+dvplNiMt+t3OV1WSKFRkEvRUKxiFD+dnUjxj3YllLFwrn7/RQGfLqQPYeOe12aSIFT0EuRcnH10nz1SDse7VCPSUt20HFwMhMXqUmaBDcFvRQ5EWEhPNqhPl8/0p7qZaPp/8lC7v0ghR37j3pdmkiBUNBLkdWgUgnGPdCGv199IbPW7qZTYjIfz9tMhtooSJBR0EuRFhpi3NO+NlMejadJ1VL89csl3Pr2XDbuPux1aSL5RkEvAtQsF8PH97bkxZ5NWbbtAF2GJjMyeb2apElQUNCL+JgZN19Wg2mDEmhXtzz/M2kFPd+YzaqdapImgU1BL3KGSqWiGNk7jldvac7WfUe55tWZDJ62muPpaqMggUlBL5IFM+Pai6owbVACVzetzNAZa7j21Vks3LzP69JEzpuCXuQcysZEMOTm5rx7ZxwHj6XT880fee7r5Rw5ke51aSI5pqAXyYErG1Zk6sB4bmtZg3dmbaDLkJn8uHa312WJ5IiCXiSHSkSF8/x1Tfn0vlaEGNz69jyeHLuY/UfVJE38W56C3sxKm9kYM1tpZivMrLWZlTWzaWa2xndbJr+KFfEHrWqX49tH4+mXUJvPU7bQaXAS05arSZr4r7zO6IcC3zrnGgIXASuAJ4EZzrl6wAzfY5GgEhUeylNdL2T8Q20pEx3BvR+k8PDHP7NbTdLED1lumzmZWUlgEVDbnfYiZrYKuNw5t8PMKgM/OOcanOu14uLiXEpKSq7qEPHaifQM3kpax6vfrSU6MpR/XNuI6y6uipl5XZoEOTNb4JyLy26/vMzoawNpwHtmttDM3jazGKCic24HgO+2wlkKvM/MUswsJS0tLQ9liHgrIiyER66qxzf923FB+RgGfraIu9//ie2/qkma+Ie8BH0Y0AJ40znXHDjMeSzTOOdGOOfinHNxsbGxeShDxD/Uq1iCMfe34ZlrGjF3/V46DU5m9NxNapImnstL0G8Ftjrn5vkejyEz+Hf5lmzw3abmrUSRwBEaYtzd7gKmDozn4uqleXr8Um4eOZcNapImHsp10DvndgJbzOy39fergOXARKCPb1sfYEKeKhQJQNXLRjO672W8fEMzVuw4QJchyQxPWkf6qQyvS5MiKNcHYwHM7GLgbSACWA/cReZ/Hp8DNYDNwI3Oub3neh0djJVgtuvAMZ4ev5Spy3fRtGopXrqhGY2qlPS6LClMGafgyB44tMv3JzXz9uAuaHw91GiZq5fN6cHYPAV9flHQS7BzzjF56U6embCUX4+c5IHL6/DwlXWJDAv1ujTJLefg+IHfQ/v0AD9z2+E0cFn8NhdRArr8G1rckasSchr0Ybl6dRE5L2ZGt6aVaV27HM99s5xXv1vL5KU7eemGZlxSU58p9Csnj8Hh1JwFePqxPz4/JByKV4TiFaBkNajS4vfHxSuedr8CRMQUypA0oxfxwA+rUvnbl0vZvv8od7apxV86NSAmUvOuApNxCo7szT64D+2CY79m/RrR5c4I7ApQvNIftxUrA4X0GQot3Yj4uUPH03n525V8MGcT1coU4989m9K+nk41zjHn4PjB81g6yeJ6AhHFzwjurGbeFSEmFkLDC3+M2VDQiwSI+Rv28uTYxazffZib4qrxt26NKBXtf6FSaNJP+JZOzjhomVWYp2fxobSQsOyDu3gFiKkAkcULf3z5SEEvEkCOnTzF0BlrGJG8nrIxETzXowldmlTyuqz8k5EBR3O4dHL0LBd3KVY2BwFeMXPpJKRoNOZV0IsEoKXb9vP4mMUs33GAbk0r8Wz3xlQoEeV1WWd31qWTM8M8Neulk/DorMP6zDCPiYWwiMIfn59T0IsEqJOnMhiRvJ6hM9ZQLDyUZ65pRM8WhdgkLf1E5pp2djPvQ6lwMotP/Fro72eVnOugZfGKAb904jUFvUiAW5t6iCfGLmbBpn3E14/lheubUK1MdO5eLCMjc0kk25n3rswllqwUK5P9zLt4xcwlliKydOI1Bb1IEMjIcIyeu4mXvl2JAU90bcjtLWsSEuKb3Z84/HtIH9x59gA/nAoZWVznNqwYlMjp0klkoY5dsqcPTIkEqlMn/3/pJORQKn2idtGj7VbmL1nJyUk7WPXdYepEHybi6G44ceiPz7eQzDNKfgvpik3OfgAzskShnfMt3lHQixQG53K+dHJkzx+eXhroGFWag6XKsvJQMabsq0L16m1p2rAeoSUq/XeAR5eFELVWkN8p6EXy4sSR7IP7t9uMLC4iHhb1e0iXrQ01WvkOXJ4x846JxcKjKAnUOniM9ycuY9KSnTQ+WpKXbmhGk6qlCn3oEji0Ri9yplPpWZx1cpYAP3Hwj8+3kMw17ewOWhavAJElc7108u3SHfx9/DL2HTlBv/ja9L+qHlHhmskXJVqjFzmdc5k9TLL7pOX/L51kMQGKKvV7SFe5+OwBHl2uUJZOujSpTOva5Xn+m+W88cM6vl22k5dvaEZcrbIF/t4SWDSjl8B28mjOl05Onfjj80MjzxLYWczGw/33g0vJq9N4atwStu8/Su9WNXmsS0OKq0la0NPplRK4TqXDkd05C/DjB7J4AfMtneQgwKNKBc1ZJ4ePp/PKlFWMmrORKqWK8ULPpiTUV5O0YKagF//iHBzbn7OZ95HdWV+kIbLkWQL7jJl3dDkILbqz2QWb9vL4mMWsSztMzxZVeeaaRpSOVvuAYKSgl8Jx8ujvvUyyXTo5/sfnh0bkbOYdUwEicvmp0CLo2MlTvPbdWoYnraN0dDj/6tGEbk0re12W5DMFveTemde3PHiupZP9WbyAQUz5nLWKjSodNEsn/mjZ9v08MXYxS7cdoEvjSvyrR2MqlPTfYw1yfhT08t/y6/qWOb5IQ9FdOvE36acyGDlzA4OnryYqLIS/X9OIGy+pVnhN0qTAKOiLivy8vuU5A7zwrm8pBWN92iGeHLuE+Rv30r5eeV64vinVy2o5LJAp6ANZvl/fMotPWnpwfUvxXkaG46N5m3hx8koc8FjnBvRuXYvQEP0MBCIFvb/R9S3Fj2z79Sh/+3IJP6xKo0WN0rzcqxl1K5Twuiw5Twr6wnLm9S3P1Sq2iF/fUvyLc47xv2zjn18t58jxU/S/qi79EuoQHqpe8oFCQZ8Xur6lFCG7Dx3nHxOX8c3iHTSsVIJXel1E02pqkhYIFPRZ0fUtRc5qyrKdPD1+KXsOn+De9rV5tIOapPm7otPULL+vb1mp2dkDXEsnEsQ6N65Eq9rleOGbFQxPWseUZTt5sWdTWtYu53VpkkeBPaOfPRSmPZP113R9S5Fcm712N0+OW8yWvUe5vVUNnujSkBJROsDvb4rG0s3mubAh+SxLJ7q+pUheHDmRzn+mrubd2RuoXDKK/7m+KVc0rOB1WXKaohH0IlLgft68jyfGLGZN6iGub16Vp69pRNkYHYPyBzkNeq1ZiMg5tahRhq/7t6P/VfX4atF2OiYm8fXi7fjDJFFyRkEvItmKDAtlUMf6fPVIO6qWKcbDHy/kvtEL2HUgi7Ya4ncU9CKSYxdWLsm4B9rw124NSV6dRofEJD6dv1mzez+noBeR8xIWGsJ98XWY8mg8jSqX5MlxS7jt7Xls3nPE69LkLBT0IpIrtcrH8Mm9rXjh+qYs3rqfTkOSeHvmek5laHbvbxT0IpJrISHGrS1rMG1QPG3qlOf5b1Zww5s/snrXQa9Lk9Mo6EUkzyqXKsY7feIYevPFbN57hKuHzWTo9DWcSM/iAjZS6BT0IpIvzIweF1dl2sB4ujapzODpq+n+2iwWbTnLNROk0OQ56M0s1MwWmtnXvscXmNk8M1tjZp+ZmT5ZIVKElCseybBbmvN27zh+PXKS69+YzQuTVnD0RBaNAqVQ5MeMfgCw4rTHLwGDnXP1gH1A33x4DxEJMB0aVWTqoHhuvqwGI5LX02VoMnPW7fG6rCIpT0FvZtWAq4G3fY8NuBIY49tlFHBdXt5DRAJXyahwXri+KR/f2xKAW0bO5alxSzhw7KTHlRUteZ3RDwEeB3474lIO+NU5l+57vBWomsf3EJEA16ZOeb4dEM998bX57KfNdEpMZsaKXV6XVWTkOujN7Bog1Tm34PTNWeya5Um1ZnafmaWYWUpaWlpuyxCRAFEsIpS/druQcQ+2pVSxcPqOSqH/JwvZc+i416UFvbzM6NsC3c1sI/ApmUs2Q4DSZvbbBU2qAduzerJzboRzLs45FxcbG5uHMkQkkFxcvTRfPdKOgR3qM3npDjoOTmbCL9vURqEA5TronXNPOeeqOedqATcD3znnbgO+B3r5dusDTMhzlSISVCLCQhjQoR7f9G9PjbLRDPj0F+4ZlcKO/Ue9Li0oFcR59E8Ag8xsLZlr9u8UwHuISBCoX7EEYx9ow9+vvpDZ63bTMTGZj+ZtIkNtFPKVLjwiIn5h854jPDluMT+u20Or2mV5sWczapWP8bosv6YLj4hIQKlRLpqP7mnJiz2bsmzbAToPSWZE8jrST6mNQl4p6EXEb5gZN19Wg2mDEmhfL5YXJq3khjd/ZOXOA16XFtAU9CLidyqVimJk70t47dbmbN13lGuGzSJx2mqOp6uNQm4o6EXEL5kZ1zSrwvRBCVx7URWGzVjDta/OYuHmfV6XFnAU9CLi18rERDD4Txfz3p2XcvBYOj3f/JHnvl7OkRPp2T9ZAAW9iASIKxpWYOrAeG5rWYN3Zm2g85BkZq/d7XVZAUFBLyIBo0RUOM9f15TP7mtFWEgIt709jyfHLmb/UTVJOxcFvYgEnJa1yzF5QHv6JdTm85QtdExMYuqynV6X5bcU9CISkKLCQ3mq64WMf6gtZWMiuG/0Ah7++Gd2q0naHyjoRSSgNauW2STtL53qM3XZLjokJvHlwq1qknYaBb2IBLzw0BAevrIekwa0o3b5GAZ+toi73v+Jbb+qSRoo6EUkiNStUIIv7m/DP65txLz1e+mUmMTouWqSpqAXkaASGmLc1fYCpg6Mp3mNMjw9fik3j5jL+rRDXpfmGQW9iASl6mWjGd33Ml7u1YyVOw/QdehMhicVzSZpCnoRCVpmxk1x1Zk+KIHLG8Ty4uSVXPfGbJZvL1pN0hT0IhL0KpSM4q074njzthbs3H+c7q/N4n+nrOLYyaLRJE1BLyJFRtemlZk+KJ4eF1flte/XcvWwmSzYtNfrsgqcgl5EipTS0RH856aLGHX3ZRw7mUGv4XN4duIyDh8P3iZpCnoRKZIS6scyZWA8vVvV5P0fN9J5SDIz16R5XVaBUNCLSJFVPDKMf/Zowhf3tyYiLIQ73pnPY18sYv+R4GqSpqAXkSLv0lplmdS/PQ9eXodxC7fRYXAS3y7d4XVZ+UZBLyJCZpO0x7s0ZMJDbYktHsn9H/7MAx8uIPXgMa9LyzMFvYjIaZpULcWEh9vyWOcGzFiZSsfEZMYsCOwmaQp6EZEzhIeG8NAVdZnUvz31KhTnL18sos97P7F13xGvS8sVBb2IyFnUrVCcz/u15p/dG5OycS+dBicz6seNAdckTUEvInIOISFGnza1mDownrhaZfnHxGXc9NYc1qYGTpM0Bb2ISA5UKxPNqLsu5T83XsSa1EN0GzqT179fy8kAaJKmoBcRySEz44ZLqjF9UAIdGlXglSmr6PHabJZu2+91aeekoBcROU+xJSJ547ZLGH57C9IOHafH67N56duVftskTUEvIpJLXZpUZvrABHo2r8qbP6yj29CZ/LTR/5qkKehFRPKgVHQ4r9x4EaP7XsaJUxncOHwOz0xYyiE/apKmoBcRyQft68Uy5dF47mpbi9FzN9F5cDI/rEr1uixAQS8ikm9iIsP4x7WNGXN/G4pFhHLnez8x6PNf2Hf4hKd1KehFRPLZJTXL8E3/djxyZV0m/rKdjoOTmLRkh2dtFBT0IiIFIDIslD93asDEh9tRuVQxHvzoZ+7/cAGpBwq/SZqCXkSkADWqUpIvH2zDk10b8sOqNDokJvF5ypZCnd0r6EVEClhYaAj3J9Rh8oD2NKxcksfHLOaOd+azZW/hNElRl23OAAAFZElEQVRT0IuIFJLascX59N5WPH9dE37Z8iudBifz1aLtBf6+CnoRkUIUEmLc3qomUwfG07ZueS4oH1Pw75nbJ5pZdTP73sxWmNkyMxvg217WzKaZ2RrfbZn8K1dEJDhUKV2Mt/vE0aRqqQJ/r7zM6NOBPzvnLgRaAQ+ZWSPgSWCGc64eMMP3WEREPJLroHfO7XDO/ey7fxBYAVQFegCjfLuNAq7La5EiIpJ7+bJGb2a1gObAPKCic24HZP5nAFTIj/cQEZHcyXPQm1lxYCzwqHPuwHk87z4zSzGzlLS0tLyWISIiZ5GnoDezcDJD/iPn3Djf5l1mVtn39cpAll19nHMjnHNxzrm42NjYvJQhIiLnkJezbgx4B1jhnEs87UsTgT6++32ACbkvT0RE8iosD89tC9wBLDGzX3zb/gq8CHxuZn2BzcCNeStRRETyItdB75ybBdhZvnxVbl9XRETyl3nVNvO/ijBLAzbl8unlgd35WE4g0JiLBo25aMjLmGs657I9yOkXQZ8XZpbinIvzuo7CpDEXDRpz0VAYY1avGxGRIKegFxEJcsEQ9CO8LsADGnPRoDEXDQU+5oBfoxcRkXMLhhm9iIicQ8AEvZl1MbNVZrbWzP7Q+tjMIs3sM9/X5/karQW0HIx5kJktN7PFZjbDzGp6UWd+ym7Mp+3Xy8ycmQX8GRo5GbOZ3eT7Xi8zs48Lu8b8loOf7Rq+610s9P18d/OizvxiZu+aWaqZLT3L183Mhvn+PhabWYt8LcA55/d/gFBgHVAbiAAWAY3O2OdBYLjv/s3AZ17XXQhjvgKI9t1/oCiM2bdfCSAZmAvEeV13IXyf6wELgTK+xxW8rrsQxjwCeMB3vxGw0eu68zjmeKAFsPQsX+8GTCbzQ6itgHn5+f6BMqO/DFjrnFvvnDsBfEpm3/vTnd4Hfwxwla8fT6DKdszOue+dc79dXXguUK2Qa8xvOfk+AzwHvAwcK8ziCkhOxnwv8Lpzbh+Acy7LRoEBJCdjdkBJ3/1SQMFfWLUAOeeSgb3n2KUH8IHLNBco/VtzyPwQKEFfFdhy2uOtvm1Z7uOcSwf2A+UKpbqCkZMxn64vmTOCQJbtmM2sOVDdOfd1YRZWgHLyfa4P1Dez2WY218y6FFp1BSMnY34WuN3MtgKTgEcKpzTPnO+/9/OSl6ZmhSmrmfmZpwvlZJ9AkuPxmNntQByQUKAVFbxzjtnMQoDBwJ2FVVAhyMn3OYzM5ZvLyfytbaaZNXHO/VrAtRWUnIz5FuB959x/zKw1MNo35oyCL88TBZpfgTKj3wpUP+1xNf74q9z/72NmYWT+uneuX5X8XU7GjJl1AP4GdHfOHS+k2gpKdmMuATQBfjCzjWSuZU4M8AOyOf3ZnuCcO+mc2wCsIjP4A1VOxtwX+BzAOTcHiCKzJ0ywytG/99wKlKD/CahnZheYWQSZB1snnrHP6X3wewHfOd9RjgCV7Zh9yxhvkRnygb5uC9mM2Tm33zlX3jlXyzlXi8zjEt2dcynelJsvcvKzPZ7MA++YWXkyl3LWF2qV+SsnY96MrwuumV1IZtAH86XoJgK9fWfftAL2O98lWfNDQCzdOOfSzexhYAqZR+zfdc4tM7N/ASnOuYlkXgRltJmtJXMmf7N3FeddDsf8ClAc+MJ33Hmzc667Z0XnUQ7HHFRyOOYpQCczWw6cAh5zzu3xruq8yeGY/wyMNLOBZC5h3BnIEzcz+4TMpbfyvuMO/wDCAZxzw8k8DtENWAscAe7K1/cP4L87ERHJgUBZuhERkVxS0IuIBDkFvYhIkFPQi4gEOQW9iEiQU9CLiAQ5Bb2ISJBT0IuIBLn/A3DhzsUvyf0pAAAAAElFTkSuQmCC\n", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "print(\"Confusion Matrix\")\n", + "print(\"{0}\".format(metrics.confusion_matrix(y_test, lf_predict_test)))\n", + "print(\"\")\n", + "\n", + "print(\"Classification Report\")\n", + "print(metrics.classification_report(y_test, lf_predict_test))\n", + "plt.plot(metrics.confusion_matrix(y_test, lf_predict_test))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Setting regularization parameter" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "first max value of 0.738 occurred at C=0.300\n" + ] + } + ], + "source": [ + "C_start = 0.1\n", + "C_end = 5\n", + "C_inc = 0.1\n", + "\n", + "C_values, recall_scores =[], []\n", + "\n", + "C_val = C_start\n", + "best_recall_score = 0\n", + "while(C_val < C_end):\n", + " C_values.append(C_val)\n", + " lr_model_loop = LogisticRegression(C=C_val, class_weight=\"balanced\", random_state=42)\n", + " lr_model_loop.fit(x_train, y_train.ravel())\n", + " lr_predict_loop_test=lr_model_loop.predict(x_test)\n", + " recall_score=metrics.recall_score(y_test, lr_predict_loop_test)\n", + " recall_scores.append(recall_score)\n", + " if(recall_score > best_recall_score):\n", + " best_recall_score = recall_score\n", + " best_lr_predict_test = lr_predict_loop_test\n", + " \n", + " C_val = C_val + C_inc\n", + " \n", + "best_score_C_val = C_values[recall_scores.index(best_recall_score)]\n", + "print(\"first max value of {0:.3f} occurred at C={1:.3f}\".format(best_recall_score, best_score_C_val))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Retrain with class_weight='balanced' and C=0.3" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "LogisticRegression(C=0.30000000000000004, class_weight='balanced', dual=False,\n", + " fit_intercept=True, intercept_scaling=1, max_iter=100,\n", + " multi_class='ovr', n_jobs=1, penalty='l2', random_state=42,\n", + " solver='liblinear', tol=0.0001, verbose=0, warm_start=False)" + ] + }, + "execution_count": 26, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from sklearn.linear_model import LogisticRegression\n", + "\n", + "lr_model = LogisticRegression(C=best_score_C_val, class_weight=\"balanced\", random_state=42)\n", + "lr_model.fit(x_train, y_train.ravel())" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Performance on Testing data" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Accurary: 0.7143\n", + "\n" + ] + } + ], + "source": [ + "# predict values using the training data\n", + "lr_predict_test = lr_model.predict(x_test)\n", + "\n", + "# import the performance metrics library\n", + "from sklearn import metrics\n", + "\n", + "# Accuracy\n", + "print(\"Accurary: {0:.4f}\".format(metrics.accuracy_score(y_test, lr_predict_test)))\n", + "print(\"\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Metrics" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Confusion Matrix\n", + "[[106 45]\n", + " [ 21 59]]\n", + "\n", + "Classification Report\n", + " precision recall f1-score support\n", + "\n", + " 0 0.83 0.70 0.76 151\n", + " 1 0.57 0.74 0.64 80\n", + "\n", + "avg / total 0.74 0.71 0.72 231\n", + "\n", + "0.7375\n" + ] + } + ], + "source": [ + "print(\"Confusion Matrix\")\n", + "print(\"{0}\".format(metrics.confusion_matrix(y_test, lr_predict_test)))\n", + "print(\"\")\n", + "\n", + "print(\"Classification Report\")\n", + "print(metrics.classification_report(y_test, lr_predict_test))\n", + "print(metrics.recall_score(y_test,lr_predict_test))" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.5" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/types/@msrvida-python-program-analysis/test/notebooks/Titanic.json b/types/@msrvida-python-program-analysis/test/notebooks/Titanic.json new file mode 100644 index 000000000000..2c09d1be326a --- /dev/null +++ b/types/@msrvida-python-program-analysis/test/notebooks/Titanic.json @@ -0,0 +1,1315 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import numpy as np\n", + "import pandas as pd\n", + "import matplotlib.pyplot as plt\n", + "df = pd.read_csv('train.csv')" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
PassengerIdSurvivedPclassNameSexAgeSibSpParchTicketFareCabinEmbarked
0103Braund, Mr. Owen Harrismale22.010A/5 211717.2500NaNS
1211Cumings, Mrs. John Bradley (Florence Briggs Th...female38.010PC 1759971.2833C85C
2313Heikkinen, Miss. Lainafemale26.000STON/O2. 31012827.9250NaNS
3411Futrelle, Mrs. Jacques Heath (Lily May Peel)female35.01011380353.1000C123S
4503Allen, Mr. William Henrymale35.0003734508.0500NaNS
\n", + "
" + ], + "text/plain": [ + " PassengerId Survived Pclass \\\n", + "0 1 0 3 \n", + "1 2 1 1 \n", + "2 3 1 3 \n", + "3 4 1 1 \n", + "4 5 0 3 \n", + "\n", + " Name Sex Age SibSp \\\n", + "0 Braund, Mr. Owen Harris male 22.0 1 \n", + "1 Cumings, Mrs. John Bradley (Florence Briggs Th... female 38.0 1 \n", + "2 Heikkinen, Miss. Laina female 26.0 0 \n", + "3 Futrelle, Mrs. Jacques Heath (Lily May Peel) female 35.0 1 \n", + "4 Allen, Mr. William Henry male 35.0 0 \n", + "\n", + " Parch Ticket Fare Cabin Embarked \n", + "0 0 A/5 21171 7.2500 NaN S \n", + "1 0 PC 17599 71.2833 C85 C \n", + "2 0 STON/O2. 3101282 7.9250 NaN S \n", + "3 0 113803 53.1000 C123 S \n", + "4 0 373450 8.0500 NaN S " + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df.head()" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "RangeIndex: 891 entries, 0 to 890\n", + "Data columns (total 12 columns):\n", + "PassengerId 891 non-null int64\n", + "Survived 891 non-null int64\n", + "Pclass 891 non-null int64\n", + "Name 891 non-null object\n", + "Sex 891 non-null object\n", + "Age 714 non-null float64\n", + "SibSp 891 non-null int64\n", + "Parch 891 non-null int64\n", + "Ticket 891 non-null object\n", + "Fare 891 non-null float64\n", + "Cabin 204 non-null object\n", + "Embarked 889 non-null object\n", + "dtypes: float64(2), int64(5), object(5)\n", + "memory usage: 83.6+ KB\n" + ] + } + ], + "source": [ + "df.info()" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "PassengerId 0\n", + "Survived 0\n", + "Pclass 0\n", + "Name 0\n", + "Sex 0\n", + "Age 177\n", + "SibSp 0\n", + "Parch 0\n", + "Ticket 0\n", + "Fare 0\n", + "Cabin 687\n", + "Embarked 2\n", + "dtype: int64" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df.apply(lambda x:sum(x.isnull()))" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "df['Age'].fillna(0 , inplace=True)\n", + "df['Embarked'].fillna('S' , inplace = True)" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "def unique(var):\n", + " unique_var=[]\n", + " for x in var:\n", + " if x not in unique_var:\n", + " unique_var.append(x)\n", + " for x in unique_var:\n", + " print(x)" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "S\n", + "C\n", + "Q\n" + ] + } + ], + "source": [ + "unique(df['Embarked'])" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "male\n", + "female\n" + ] + } + ], + "source": [ + "unique(df['Sex'])" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [], + "source": [ + "df['Sex'] = df['Sex'].map({'female':0,'male':1}).astype(np.int)" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [], + "source": [ + "df['Embarked'] = df['Embarked'].map({'nan':0,'S':1,'C':2,'Q':3}).astype(np.int)" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "RangeIndex: 891 entries, 0 to 890\n", + "Data columns (total 12 columns):\n", + "PassengerId 891 non-null int64\n", + "Survived 891 non-null int64\n", + "Pclass 891 non-null int64\n", + "Name 891 non-null object\n", + "Sex 891 non-null int32\n", + "Age 891 non-null float64\n", + "SibSp 891 non-null int64\n", + "Parch 891 non-null int64\n", + "Ticket 891 non-null object\n", + "Fare 891 non-null float64\n", + "Cabin 204 non-null object\n", + "Embarked 891 non-null int32\n", + "dtypes: float64(2), int32(2), int64(5), object(3)\n", + "memory usage: 76.6+ KB\n" + ] + } + ], + "source": [ + "df.info()" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [], + "source": [ + "del df['Name']" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [], + "source": [ + "del df['Ticket']\n", + "#del df['Cabin']" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
PassengerIdSurvivedPclassSexAgeSibSpParchFareCabinEmbarked
0103122.0107.2500NaN1
1211038.01071.2833C852
2313026.0007.9250NaN1
3411035.01053.1000C1231
4503135.0008.0500NaN1
\n", + "
" + ], + "text/plain": [ + " PassengerId Survived Pclass Sex Age SibSp Parch Fare Cabin \\\n", + "0 1 0 3 1 22.0 1 0 7.2500 NaN \n", + "1 2 1 1 0 38.0 1 0 71.2833 C85 \n", + "2 3 1 3 0 26.0 0 0 7.9250 NaN \n", + "3 4 1 1 0 35.0 1 0 53.1000 C123 \n", + "4 5 0 3 1 35.0 0 0 8.0500 NaN \n", + "\n", + " Embarked \n", + "0 1 \n", + "1 2 \n", + "2 1 \n", + "3 1 \n", + "4 1 " + ] + }, + "execution_count": 14, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df.head()" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [], + "source": [ + "feature=['PassengerId','Pclass','Sex','Age','SibSp','Parch','Fare','Embarked']\n", + "x=df[feature]" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "0 0\n", + "1 1\n", + "2 1\n", + "3 1\n", + "4 0\n", + "Name: Survived, dtype: int64" + ] + }, + "execution_count": 16, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "y=df['Survived']\n", + "y.head()" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "0 549\n", + "1 342\n", + "Name: Survived, dtype: int64" + ] + }, + "execution_count": 17, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df['Survived'].value_counts()" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "C:\\Users\\AMR\\Anaconda3\\lib\\site-packages\\sklearn\\cross_validation.py:41: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. Also note that the interface of the new CV iterators are different from that of this module. This module will be removed in 0.20.\n", + " \"This module will be removed in 0.20.\", DeprecationWarning)\n" + ] + } + ], + "source": [ + "from sklearn.cross_validation import train_test_split\n", + "x_train,x_test,y_train,y_test=train_test_split(x,y,test_size=0.25,random_state=6)" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "(668, 8)\n", + "(223, 8)\n", + "(668,)\n", + "(223,)\n" + ] + } + ], + "source": [ + "print(x_train.shape) \n", + "print(x_test.shape)\n", + "print(y_train.shape)\n", + "print(y_test.shape)" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "LinearRegression(copy_X=True, fit_intercept=True, n_jobs=1, normalize=False)" + ] + }, + "execution_count": 20, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "#Linear Regression\n", + "from sklearn.linear_model import LinearRegression\n", + "linreg=LinearRegression()\n", + "linreg.fit(x_train,y_train)" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "0.5160966104067846\n" + ] + } + ], + "source": [ + "y_pred = linreg.predict(x_test)\n", + "from sklearn import metrics\n", + "pred_accuracy = np.sqrt(metrics.mean_absolute_error(y_test,y_pred))\n", + "print(pred_accuracy)" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "0.8475336322869955\n" + ] + } + ], + "source": [ + "#Logistic Regression\n", + "from sklearn.linear_model import LogisticRegression\n", + "logreg=LogisticRegression()\n", + "logreg.fit(x_train,y_train)\n", + "y_pred=logreg.predict(x_test)\n", + "print(metrics.accuracy_score(y_test,y_pred))" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "0.8475336322869955\n" + ] + } + ], + "source": [ + "from sklearn.neighbors import KNeighborsClassifier\n", + "knn=KNeighborsClassifier(n_neighbors=2)\n", + "knn.fit(x_train,y_train)\n", + "pred=knn.predict(x_test)\n", + "print(metrics.accuracy_score(y_test,y_pred))" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "metadata": {}, + "outputs": [], + "source": [ + "df_test=pd.read_csv('test.csv')" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
PassengerIdPclassSexAgeSibSpParchFareEmbarkedSurvived
08923134.5007.829230
18933047.0107.000011
28942162.0009.687530
38953127.0008.662510
48963022.01112.287511
\n", + "
" + ], + "text/plain": [ + " PassengerId Pclass Sex Age SibSp Parch Fare Embarked Survived\n", + "0 892 3 1 34.5 0 0 7.8292 3 0\n", + "1 893 3 0 47.0 1 0 7.0000 1 1\n", + "2 894 2 1 62.0 0 0 9.6875 3 0\n", + "3 895 3 1 27.0 0 0 8.6625 1 0\n", + "4 896 3 0 22.0 1 1 12.2875 1 1" + ] + }, + "execution_count": 34, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df_test.head()" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "RangeIndex: 418 entries, 0 to 417\n", + "Data columns (total 11 columns):\n", + "PassengerId 418 non-null int64\n", + "Pclass 418 non-null int64\n", + "Name 418 non-null object\n", + "Sex 418 non-null object\n", + "Age 332 non-null float64\n", + "SibSp 418 non-null int64\n", + "Parch 418 non-null int64\n", + "Ticket 418 non-null object\n", + "Fare 417 non-null float64\n", + "Cabin 91 non-null object\n", + "Embarked 418 non-null object\n", + "dtypes: float64(2), int64(4), object(5)\n", + "memory usage: 36.0+ KB\n" + ] + } + ], + "source": [ + "df_test.info()" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "metadata": {}, + "outputs": [], + "source": [ + "df_test['Age'].fillna(0 , inplace=True)" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "metadata": {}, + "outputs": [], + "source": [ + "del df_test['Ticket']\n", + "del df_test['Cabin']\n", + "del df_test['Name']" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "RangeIndex: 418 entries, 0 to 417\n", + "Data columns (total 8 columns):\n", + "PassengerId 418 non-null int64\n", + "Pclass 418 non-null int64\n", + "Sex 418 non-null object\n", + "Age 418 non-null float64\n", + "SibSp 418 non-null int64\n", + "Parch 418 non-null int64\n", + "Fare 417 non-null float64\n", + "Embarked 418 non-null object\n", + "dtypes: float64(2), int64(4), object(2)\n", + "memory usage: 26.2+ KB\n" + ] + } + ], + "source": [ + "df_test.info()" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "metadata": {}, + "outputs": [], + "source": [ + "df_test['Sex'] = df_test['Sex'].map({'female':0,'male':1}).astype(np.int)\n", + "df_test['Embarked'] = df_test['Embarked'].map({'nan':0,'S':1,'C':2,'Q':3}).astype(np.int)\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "metadata": {}, + "outputs": [], + "source": [ + "df_test['Fare'].fillna(0 , inplace=True)" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "metadata": {}, + "outputs": [], + "source": [ + "test_feat = ['PassengerId','Pclass','Sex','Age','SibSp','Parch','Fare','Embarked']\n", + "X_test = df_test[test_feat]" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "metadata": {}, + "outputs": [], + "source": [ + "Y_test_pred=logreg.predict(X_test)" + ] + }, + { + "cell_type": "code", + "execution_count": 39, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
PassengerIdSurvived
08920
18931
28940
38950
48961
58970
68981
78990
89001
99010
109020
119030
129041
139050
149061
159071
169080
179090
189101
199111
209120
219130
229141
239151
249161
259170
269181
279190
289200
299210
309220
319230
329241
339251
349260
359270
369281
379291
389300
399310
409320
419330
429340
439351
449361
459370
469380
479390
489401
499411
\n", + "
" + ], + "text/plain": [ + " PassengerId Survived\n", + "0 892 0\n", + "1 893 1\n", + "2 894 0\n", + "3 895 0\n", + "4 896 1\n", + "5 897 0\n", + "6 898 1\n", + "7 899 0\n", + "8 900 1\n", + "9 901 0\n", + "10 902 0\n", + "11 903 0\n", + "12 904 1\n", + "13 905 0\n", + "14 906 1\n", + "15 907 1\n", + "16 908 0\n", + "17 909 0\n", + "18 910 1\n", + "19 911 1\n", + "20 912 0\n", + "21 913 0\n", + "22 914 1\n", + "23 915 1\n", + "24 916 1\n", + "25 917 0\n", + "26 918 1\n", + "27 919 0\n", + "28 920 0\n", + "29 921 0\n", + "30 922 0\n", + "31 923 0\n", + "32 924 1\n", + "33 925 1\n", + "34 926 0\n", + "35 927 0\n", + "36 928 1\n", + "37 929 1\n", + "38 930 0\n", + "39 931 0\n", + "40 932 0\n", + "41 933 0\n", + "42 934 0\n", + "43 935 1\n", + "44 936 1\n", + "45 937 0\n", + "46 938 0\n", + "47 939 0\n", + "48 940 1\n", + "49 941 1" + ] + }, + "execution_count": 39, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df_test['Survived']=Y_test_pred\n", + "\n", + "df_result = df_test.drop(['Pclass','Sex','Age','SibSp','Parch','Fare','Embarked'], axis=1)\n", + "#df_result['Gender'] = df_test['Sex'].map({1:'Male',0:'Female'})\n", + "df_result['Survived'] = df_result['Survived']\n", + "df_result.to_csv('result.csv', index=False)\n", + "df_result.head(50)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.4" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/types/@msrvida-python-program-analysis/test/notebooks/Titanic2.json b/types/@msrvida-python-program-analysis/test/notebooks/Titanic2.json new file mode 100644 index 000000000000..73ce2797add7 --- /dev/null +++ b/types/@msrvida-python-program-analysis/test/notebooks/Titanic2.json @@ -0,0 +1,464 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import numpy as np\n", + "import pandas as pd\n", + "import matplotlib.pyplot as plt\n", + "df = pd.read_csv('train.csv')" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
PassengerIdSurvivedPclassNameSexAgeSibSpParchTicketFareCabinEmbarked
0103Braund, Mr. Owen Harrismale22.010A/5 211717.2500NaNS
1211Cumings, Mrs. John Bradley (Florence Briggs Th...female38.010PC 1759971.2833C85C
2313Heikkinen, Miss. Lainafemale26.000STON/O2. 31012827.9250NaNS
3411Futrelle, Mrs. Jacques Heath (Lily May Peel)female35.01011380353.1000C123S
4503Allen, Mr. William Henrymale35.0003734508.0500NaNS
\n
", + "text/plain": " PassengerId Survived Pclass \\\n0 1 0 3 \n1 2 1 1 \n2 3 1 3 \n3 4 1 1 \n4 5 0 3 \n\n Name Sex Age SibSp \\\n0 Braund, Mr. Owen Harris male 22.0 1 \n1 Cumings, Mrs. John Bradley (Florence Briggs Th... female 38.0 1 \n2 Heikkinen, Miss. Laina female 26.0 0 \n3 Futrelle, Mrs. Jacques Heath (Lily May Peel) female 35.0 1 \n4 Allen, Mr. William Henry male 35.0 0 \n\n Parch Ticket Fare Cabin Embarked \n0 0 A/5 21171 7.2500 NaN S \n1 0 PC 17599 71.2833 C85 C \n2 0 STON/O2. 3101282 7.9250 NaN S \n3 0 113803 53.1000 C123 S \n4 0 373450 8.0500 NaN S " + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df.head()" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": "\nRangeIndex: 891 entries, 0 to 890\nData columns (total 12 columns):\nPassengerId 891 non-null int64\nSurvived 891 non-null int64\nPclass 891 non-null int64\nName 891 non-null object\nSex 891 non-null object\nAge 714 non-null float64\nSibSp 891 non-null int64\nParch 891 non-null int64\nTicket 891 non-null object\nFare 891 non-null float64\nCabin 204 non-null object\nEmbarked 889 non-null object\ndtypes: float64(2), int64(5), object(5)\nmemory usage: 83.6+ KB\n" + } + ], + "source": [ + "df.info()" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": "PassengerId 0\nSurvived 0\nPclass 0\nName 0\nSex 0\nAge 177\nSibSp 0\nParch 0\nTicket 0\nFare 0\nCabin 687\nEmbarked 2\ndtype: int64" + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df.apply(lambda x:sum(x.isnull()))" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "df['Age'].fillna(0 , inplace=True)\n", + "df['Embarked'].fillna('S' , inplace = True)" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "def unique(var):\n", + " unique_var=[]\n", + " for x in var:\n", + " if x not in unique_var:\n", + " unique_var.append(x)\n", + " for x in unique_var:\n", + " print(x)" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": "S\nC\nQ\n" + } + ], + "source": [ + "unique(df['Embarked'])" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": "male\nfemale\n" + } + ], + "source": [ + "unique(df['Sex'])" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [], + "source": [ + "df['Sex'] = df['Sex'].map({'female':0,'male':1}).astype(np.int)" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [], + "source": [ + "df['Embarked'] = df['Embarked'].map({'nan':0,'S':1,'C':2,'Q':3}).astype(np.int)" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": "\nRangeIndex: 891 entries, 0 to 890\nData columns (total 12 columns):\nPassengerId 891 non-null int64\nSurvived 891 non-null int64\nPclass 891 non-null int64\nName 891 non-null object\nSex 891 non-null int32\nAge 891 non-null float64\nSibSp 891 non-null int64\nParch 891 non-null int64\nTicket 891 non-null object\nFare 891 non-null float64\nCabin 204 non-null object\nEmbarked 891 non-null int32\ndtypes: float64(2), int32(2), int64(5), object(3)\nmemory usage: 76.6+ KB\n" + } + ], + "source": [ + "df.info()" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [], + "source": [ + "del df['Name']" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [], + "source": [ + "del df['Ticket']\n", + "#del df['Cabin']" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
PassengerIdSurvivedPclassSexAgeSibSpParchFareCabinEmbarked
0103122.0107.2500NaN1
1211038.01071.2833C852
2313026.0007.9250NaN1
3411035.01053.1000C1231
4503135.0008.0500NaN1
\n
", + "text/plain": " PassengerId Survived Pclass Sex Age SibSp Parch Fare Cabin \\\n0 1 0 3 1 22.0 1 0 7.2500 NaN \n1 2 1 1 0 38.0 1 0 71.2833 C85 \n2 3 1 3 0 26.0 0 0 7.9250 NaN \n3 4 1 1 0 35.0 1 0 53.1000 C123 \n4 5 0 3 1 35.0 0 0 8.0500 NaN \n\n Embarked \n0 1 \n1 2 \n2 1 \n3 1 \n4 1 " + }, + "execution_count": 14, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df.head()" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [], + "source": [ + "feature=['PassengerId','Pclass','Sex','Age','SibSp','Parch','Fare','Embarked']\n", + "x=df[feature]" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": "0 0\n1 1\n2 1\n3 1\n4 0\nName: Survived, dtype: int64" + }, + "execution_count": 16, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "y=df['Survived']\n", + "y.head()" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": "0 549\n1 342\nName: Survived, dtype: int64" + }, + "execution_count": 17, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df['Survived'].value_counts()" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [ + { + "ename": "ModuleNotFoundError", + "evalue": "No module named 'sklearn.cross_validation'", + "output_type": "error", + "traceback": [ + "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[1;31mModuleNotFoundError\u001b[0m Traceback (most recent call last)", + "\u001b[1;32m\u001b[0m in \u001b[0;36m\u001b[1;34m\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[1;32mfrom\u001b[0m \u001b[0msklearn\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcross_validation\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mtrain_test_split\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 2\u001b[0m \u001b[0mx_train\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mx_test\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0my_train\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0my_test\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mtrain_test_split\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mx\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0my\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mtest_size\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m0.25\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mrandom_state\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m6\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n", + "\u001b[1;31mModuleNotFoundError\u001b[0m: No module named 'sklearn.cross_validation'" + ] + } + ], + "source": [ + "from sklearn.model_selection import train_test_split\n", + "x_train,x_test,y_train,y_test=train_test_split(x,y,test_size=0.25,random_state=6)" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": {}, + "outputs": [], + "source": [ + "print(x_train.shape) \n", + "print(x_test.shape)\n", + "print(y_train.shape)\n", + "print(y_test.shape)" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": {}, + "outputs": [], + "source": [ + "#Linear Regression\n", + "from sklearn.linear_model import LinearRegression\n", + "linreg=LinearRegression()\n", + "linreg.fit(x_train,y_train)" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": {}, + "outputs": [], + "source": [ + "y_pred = linreg.predict(x_test)\n", + "from sklearn import metrics\n", + "pred_accuracy = np.sqrt(metrics.mean_absolute_error(y_test,y_pred))\n", + "print(pred_accuracy)" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": {}, + "outputs": [], + "source": [ + "#Logistic Regression\n", + "from sklearn.linear_model import LogisticRegression\n", + "logreg=LogisticRegression()\n", + "logreg.fit(x_train,y_train)\n", + "y_pred=logreg.predict(x_test)\n", + "print(metrics.accuracy_score(y_test,y_pred))" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": {}, + "outputs": [], + "source": [ + "from sklearn.neighbors import KNeighborsClassifier\n", + "knn=KNeighborsClassifier(n_neighbors=2)\n", + "knn.fit(x_train,y_train)\n", + "pred=knn.predict(x_test)\n", + "print(metrics.accuracy_score(y_test,y_pred))" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": {}, + "outputs": [], + "source": [ + "df_test=pd.read_csv('test.csv')" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": {}, + "outputs": [], + "source": [ + "df_test.head()" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": {}, + "outputs": [], + "source": [ + "df_test.info()" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": {}, + "outputs": [], + "source": [ + "df_test['Age'].fillna(0 , inplace=True)" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": {}, + "outputs": [], + "source": [ + "del df_test['Ticket']\n", + "del df_test['Cabin']\n", + "del df_test['Name']" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": {}, + "outputs": [], + "source": [ + "df_test.info()" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": {}, + "outputs": [], + "source": [ + "df_test['Sex'] = df_test['Sex'].map({'female':0,'male':1}).astype(np.int)\n", + "df_test['Embarked'] = df_test['Embarked'].map({'nan':0,'S':1,'C':2,'Q':3}).astype(np.int)\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": {}, + "outputs": [], + "source": [ + "df_test['Fare'].fillna(0 , inplace=True)" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": {}, + "outputs": [], + "source": [ + "test_feat = ['PassengerId','Pclass','Sex','Age','SibSp','Parch','Fare','Embarked']\n", + "X_test = df_test[test_feat]" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": {}, + "outputs": [], + "source": [ + "Y_test_pred=logreg.predict(X_test)" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": {}, + "outputs": [], + "source": [ + "df_test['Survived']=Y_test_pred\n", + "\n", + "df_result = df_test.drop(['Pclass','Sex','Age','SibSp','Parch','Fare','Embarked'], axis=1)\n", + "#df_result['Gender'] = df_test['Sex'].map({1:'Male',0:'Female'})\n", + "df_result['Survived'] = df_result['Survived']\n", + "df_result.to_csv('result.csv', index=False)\n", + "df_result.head(50)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.8" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/types/@msrvida-python-program-analysis/test/notebooks/evaluating-models-expected.json b/types/@msrvida-python-program-analysis/test/notebooks/evaluating-models-expected.json new file mode 100644 index 000000000000..e46ca4d28e91 --- /dev/null +++ b/types/@msrvida-python-program-analysis/test/notebooks/evaluating-models-expected.json @@ -0,0 +1,165 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": "C:\\Users\\jimgries\\AppData\\Local\\Continuum\\anaconda3\\envs\\ml-workflows\\lib\\site-packages\\pyarrow\\pandas_compat.py:708: FutureWarning: .labels was deprecated in version 0.24.0. Use .codes instead.\n labels = getattr(columns, 'labels', None) or [\nC:\\Users\\jimgries\\AppData\\Local\\Continuum\\anaconda3\\envs\\ml-workflows\\lib\\site-packages\\pyarrow\\pandas_compat.py:735: FutureWarning: the 'labels' keyword is deprecated, use 'codes' instead\n return pd.MultiIndex(levels=new_levels, labels=labels, names=columns.names)\nC:\\Users\\jimgries\\AppData\\Local\\Continuum\\anaconda3\\envs\\ml-workflows\\lib\\site-packages\\pyarrow\\pandas_compat.py:752: FutureWarning: .labels was deprecated in version 0.24.0. Use .codes instead.\n labels, = index.labels\n" + } + ], + "source": [ + "import pandas as pd\n", + "data = pd.read_parquet(\"data/training.parquet\")\n", + "" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "def model_accuracy(osm, df):\n", + " correct = 0\n", + " incorrect = 0\n", + " for row in df.itertuples(): \n", + " if row.label == osm.predict(row.text):\n", + " correct += 1\n", + " else:\n", + " incorrect += 1\n", + " \n", + " if correct + incorrect == 0:\n", + " return 100\n", + " \n", + " return (float(correct) / float(correct + incorrect) * 100)\n", + "" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "legit_sample = data[data.label == 'legitimate'].sample(2000)\n", + "spam_sample = data[data.label == 'spam'].sample(18000)\n", + "unbalanced = pd.DataFrame.append(legit_sample, spam_sample)\n", + "" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "from sklearn.model_selection import train_test_split\n", + "unbalanced_train, unbalanced_test = train_test_split(unbalanced, test_size=0.3)\n", + "" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "from collections import defaultdict\n", + "import re\n", + " \n", + "class SensitiveSpamModel(object):\n", + " \n", + " def __init__(self):\n", + " self.legit = set()\n", + " self.spam = set()\n", + " \n", + " def fit(self, df):\n", + " \"\"\" Train a model based on the most frequent unique \n", + " words in each class of documents \"\"\"\n", + " legit_words = defaultdict(lambda: 0)\n", + " spam_words = defaultdict(lambda: 0)\n", + " \n", + " for tup in df.itertuples():\n", + " target = spam_words\n", + " if tup.label == \"legitimate\":\n", + " target = legit_words\n", + " for word in re.split(r\"\\W+\", tup.text):\n", + " if len(word) > 0:\n", + " target[word.lower()] += 1\n", + " \n", + " # remove words common to both classes\n", + " for word in set(legit_words.keys()).intersection(set(spam_words.keys())):\n", + " del legit_words[word]\n", + " del spam_words[word]\n", + " \n", + " top_legit_words = sorted(legit_words.items(), key=lambda kv: kv[1], reverse=True)\n", + " top_spam_words = sorted(spam_words.items(), key=lambda kv: kv[1], reverse=True)\n", + " \n", + " # store ten times as many words from the spam set\n", + " self.legit = set([t[0] for t in top_legit_words[:100]])\n", + " self.spam = set([t[0] for t in top_spam_words[:1000]])\n", + " \n", + " def predict(self, text):\n", + " legit_score = 0\n", + " spam_score = 0\n", + " \n", + " for word in re.split(r\"\\W+\", text):\n", + " w = word.lower()\n", + " if word in self.legit:\n", + " legit_score = legit_score + 1\n", + " elif word in self.spam:\n", + " spam_score = spam_score + 1\n", + " \n", + " # bias results towards spam in the event of ties\n", + " return (legit_score > spam_score) and \"legitimate\" or \"spam\"\n", + "\n", + "ssm = SensitiveSpamModel()\n", + "ssm.fit(unbalanced_train)\n", + "" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": "63.582499999999996" + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "model_accuracy(ssm, data)\n", + "\n", + "\n", + "" + ] + } + ], + "nbformat": 4, + "nbformat_minor": 2, + "metadata": { + "language_info": { + "name": "python", + "codemirror_mode": { + "name": "ipython", + "version": 3 + } + }, + "orig_nbformat": 2, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "npconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": 3 + } +} diff --git a/types/@msrvida-python-program-analysis/test/notebooks/evaluating-models.json b/types/@msrvida-python-program-analysis/test/notebooks/evaluating-models.json new file mode 100644 index 000000000000..24669d81f221 --- /dev/null +++ b/types/@msrvida-python-program-analysis/test/notebooks/evaluating-models.json @@ -0,0 +1,174 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "source": [ + "import pandas as pd\n", + "data = pd.read_parquet(\"data/training.parquet\")\n", + "" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "source": [ + "def model_accuracy(osm, df):\n", + " correct = 0\n", + " incorrect = 0\n", + " for row in df.itertuples(): \n", + " if row.label == osm.predict(row.text):\n", + " correct += 1\n", + " else:\n", + " incorrect += 1\n", + " \n", + " if correct + incorrect == 0:\n", + " return 100\n", + " \n", + " return (float(correct) / float(correct + incorrect) * 100)\n", + "" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "source": [ + "legit_sample = data[data.label == 'legitimate'].sample(2000)\n", + "spam_sample = data[data.label == 'spam'].sample(18000)\n", + "unbalanced = pd.DataFrame.append(legit_sample, spam_sample)\n", + "" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + " To avoid overfitting, we'll split the unbalanced data set into training and test sets, using functionality from scikit-learn." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "source": [ + "from sklearn.model_selection import train_test_split\n", + "unbalanced_train, unbalanced_test = train_test_split(unbalanced, test_size=0.3)\n", + "" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + " We'll now create a simple model that should work pretty well for spam messages but not necessarily as well for legitimate ones." + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "source": [ + "from collections import defaultdict\n", + "import re\n", + " \n", + "class SensitiveSpamModel(object):\n", + " \n", + " def __init__(self):\n", + " self.legit = set()\n", + " self.spam = set()\n", + " \n", + " def fit(self, df):\n", + " \"\"\" Train a model based on the most frequent unique \n", + " words in each class of documents \"\"\"\n", + " legit_words = defaultdict(lambda: 0)\n", + " spam_words = defaultdict(lambda: 0)\n", + " \n", + " for tup in df.itertuples():\n", + " target = spam_words\n", + " if tup.label == \"legitimate\":\n", + " target = legit_words\n", + " for word in re.split(r\"\\W+\", tup.text):\n", + " if len(word) > 0:\n", + " target[word.lower()] += 1\n", + " \n", + " # remove words common to both classes\n", + " for word in set(legit_words.keys()).intersection(set(spam_words.keys())):\n", + " del legit_words[word]\n", + " del spam_words[word]\n", + " \n", + " top_legit_words = sorted(legit_words.items(), key=lambda kv: kv[1], reverse=True)\n", + " top_spam_words = sorted(spam_words.items(), key=lambda kv: kv[1], reverse=True)\n", + " \n", + " # store ten times as many words from the spam set\n", + " self.legit = set([t[0] for t in top_legit_words[:100]])\n", + " self.spam = set([t[0] for t in top_spam_words[:1000]])\n", + " \n", + " def predict(self, text):\n", + " legit_score = 0\n", + " spam_score = 0\n", + " \n", + " for word in re.split(r\"\\W+\", text):\n", + " w = word.lower()\n", + " if word in self.legit:\n", + " legit_score = legit_score + 1\n", + " elif word in self.spam:\n", + " spam_score = spam_score + 1\n", + " \n", + " # bias results towards spam in the event of ties\n", + " return (legit_score > spam_score) and \"legitimate\" or \"spam\"\n", + "\n", + "ssm = SensitiveSpamModel()\n", + "ssm.fit(unbalanced_train)\n", + "" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "source": [ + "model_accuracy(ssm, unbalanced_train)\n", + "" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "source": [ + "model_accuracy(ssm, unbalanced_test)\n", + "" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "source": [ + "model_accuracy(ssm, data)\n", + "" + ] + } + ], + "nbformat": 4, + "nbformat_minor": 2, + "metadata": { + "language_info": { + "name": "python", + "codemirror_mode": { + "name": "ipython", + "version": 3 + } + }, + "orig_nbformat": 2, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "npconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": 3 + } +} diff --git a/types/@msrvida-python-program-analysis/test/notebooks/index.d.ts b/types/@msrvida-python-program-analysis/test/notebooks/index.d.ts new file mode 100644 index 000000000000..96ff2b1f63ef --- /dev/null +++ b/types/@msrvida-python-program-analysis/test/notebooks/index.d.ts @@ -0,0 +1,17 @@ +export interface Notebook { + cells: Cell[]; +} +export interface Cell { + cell_type: 'code' | 'markdown'; + execution_count: number; + source: string[]; +} +export declare function cellCode(nb: Notebook): string[]; +export declare const vvNotebook: Notebook; +export declare const titanicNotebook: Notebook; +export declare const titanicNotebook2: Notebook; +export declare const pimaNotebook: Notebook; +export declare const evalModelsNotebook: Notebook; +export declare const evalModelsExpectedNotebook: Notebook; +export declare const featureEngineeringNotebook: Notebook; +export declare const featureEngineeringExpectedNotebook: Notebook; diff --git a/types/@msrvida-python-program-analysis/test/notebooks/vv.json b/types/@msrvida-python-program-analysis/test/notebooks/vv.json new file mode 100644 index 000000000000..741e96421479 --- /dev/null +++ b/types/@msrvida-python-program-analysis/test/notebooks/vv.json @@ -0,0 +1,306 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Let's take a look at a simple way to trying to identify some structure in our data. Getting some understanding of the data is an important first step before we even start to look at using machine learning techniques to train a model; in this notebook, we'll approach that problem from a couple of different angles.\n", + "\n", + "We'll start by loading our training data." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "C:\\Users\\jimgries\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\pyarrow\\pandas_compat.py:708: FutureWarning: .labels was deprecated in version 0.24.0. Use .codes instead.\n", + " labels = getattr(columns, 'labels', None) or [\n", + "C:\\Users\\jimgries\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\pyarrow\\pandas_compat.py:735: FutureWarning: the 'labels' keyword is deprecated, use 'codes' instead\n", + " return pd.MultiIndex(levels=new_levels, labels=labels, names=columns.names)\n", + "C:\\Users\\jimgries\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\pyarrow\\pandas_compat.py:752: FutureWarning: .labels was deprecated in version 0.24.0. Use .codes instead.\n", + " labels, = index.labels\n" + ] + } + ], + "source": [ + "import pandas as pd\n", + "data = pd.read_parquet(\"data/training.parquet\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Our training data (which we generated in [the previous notebook](00-generator.ipynb)) consists of labels (either `legitimate` or `spam`) and short documents of plausible English text. We can inspect these data:" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "scrolled": true + }, + "source": [ + "data.sample(50)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Ultimately, machine learning algorithms operate on data that is structured differently than the data we might deal with in database tables or application programs. In order to identify and exploit structure in these data, we are going to convert our natural-language documents to points in space by converting them to vectors of floating-point numbers.\n", + "\n", + "This process is often tricky, since you want a way to map from arbitrary data to some points in (some) space that preserves the structure of the data. That is, documents that are similar should map to points that are similar (for some definition of similarity), and documents that are dissimilar should not map to similar points. The name for this process of turning real-world data into a form that a machine learning algorithm can take advantage of is *feature engineering*. \n", + "\n", + "You'll learn more about feature engineering in the next notebook; for now, we'll just take a very basic approach that will let us visualize our data. We'll first convert our documents to *k-shingles*, or sequences of *k* characters (for some small value of *k*). This means that a document like\n", + "\n", + "`the quick brown fox jumps over the lazy dog`\n", + "\n", + "would become this sequence of 4-shingles: \n", + "\n", + "`['the ', 'he q', 'e qu', ' qui', 'quic', 'uick', 'ick ', 'ck b', 'k br', ' bro', 'brow', 'rown', 'own ', 'wn f', 'n fo', ' fox', 'fox ', 'ox j', 'x ju', ' jum', 'jump', 'umps', 'mps ', 'ps o', 's ov', ' ove', 'over', 'ver ', 'er t', 'r th', ' the', 'the ', 'he l', 'e la', ' laz', 'lazy', 'azy ', 'zy d', 'y do', ' dog']`\n", + "\n", + "Shingling gets us a step closer to having vector representations of documents -- ultimately, our assumption is that spam documents will have some k-shingles that legitimate documents don't, and vice versa. Here's how we'd add a field of shingles to our data:" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "def doc2shingles(k):\n", + " def kshingles(doc):\n", + " return [doc[i:i + k] for i in range(len(doc) - k + 1)]\n", + " return kshingles\n", + "\n", + "data[\"shingles\"] = data[\"text\"].apply(doc2shingles(4))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Remember, our goal is to be able to learn a function that can separate between documents that are likely to represent legitimate messages (i.e., prose in the style of Jane Austen) or spam messages (i.e., prose in the style of food-product reviews), so we'll still want to transform our lists of shingles into vectors.\n", + "\n", + "1. We'll collect shingle counts for each example, showing us how frequent each shingle is in a given document;\n", + "2. We'll then turn those raw counts into frequencies (i.e., for a given shingle what percentage of shingle in given document are that word?), giving us a mapping from shingles to frequencies for each document;\n", + "3. Finally, we'll encode these mappings as fixed-size vectors in a space-efficient way, by using a hash function to determine which vector element should get a given frequency. Hashing has a few advantages, but for our purposes the most important advantage is that we don't need to know all of the shingles we might see in advance. \n", + "\n", + "(That's what we'll _logically_ do -- we'll _actually_ do these steps a bit out of order because it will make our code simpler and more efficient without changing the results.)" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "import numpy as np\n", + "\n", + "def hashing_frequency(vecsize, h):\n", + " \"\"\" \n", + " returns a function that will collect shingle frequencies \n", + " into a vector with _vecsize_ elements and will use \n", + " the hash function _h_ to choose which vector element \n", + " to update for a given term\n", + " \"\"\"\n", + " \n", + " def hf(words):\n", + " if type(words) is type(\"\"):\n", + " # handle both lists of words and space-delimited strings\n", + " words = words.split(\" \")\n", + " \n", + " result = np.zeros(vecsize)\n", + " for term in words:\n", + " result[h(term) % vecsize] += 1.0\n", + " \n", + " total = sum(result)\n", + " for i in range(len(result)):\n", + " result[i] /= total\n", + "\n", + " return result\n", + " \n", + " return hf" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "a = np.array([hashing_frequency(1024, hash)(v) for v in data[\"shingles\"].values])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "So now instead of having documents (which we had from the raw data) or lists of shingles, we have vectors representing shingle frequencies. Because we've hashed shingles into these vectors, we can't in general reconstruct a document or the shingles from a vector, but we *do* know that if the same shingle appears in two documents, their vectors will reflect it in corresponding buckets.\n", + "\n", + "However, we've generated a 1024-element vector. Recall that our ultimate goal is to place documents in space so that we can identify a way to separate legitimate documents from spam documents. Our 1024-element vector is a point in a space, but it's a point in a space that most of our geometric intuitions don't apply to (some of us have enough trouble navigating the three dimensions of the physical world). \n", + "\n", + "Let's use a very basic technique to project these vectors to a much smaller space that we can visualize. [Principal component analysis](https://en.wikipedia.org/wiki/Principal_component_analysis), or PCA, is a statistical technique that is over a century old; it takes observations in a high-dimensional space (like our 1024-element vectors) and maps them to a (potentially much) smaller number of dimensions. It's an elegant technique, and the most important things to know about it are that it tries to ensure that the dimensions that have the most variance contribute the most to the mapping, while the dimensions with the least variance are (more-or-less) disregarded. The other important thing to know about PCA is that there are very efficient ways to compute it, even on large datasets that don't fit in memory on a single machine. We'll see it in action now, using the [implementation from scikit-learn](https://scikit-learn.org/stable/modules/generated/sklearn.decomposition.PCA.html#sklearn.decomposition.PCA)." + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "import sklearn.decomposition\n", + "\n", + "DIMENSIONS = 2\n", + "\n", + "pca2 = sklearn.decomposition.PCA(DIMENSIONS)\n", + "\n", + "pca_a = pca2.fit_transform(a)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The `.fit_transform()` method takes an array of high-dimensional observations and will both perform the principal component analysis (the \"fit\" part) and use that to map the high-dimensional values to low-dimensional ones (the \"transform\" part). We can see what the transformed vectors look like:" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "source": [ + "pca_df = pd.DataFrame(pca_a, columns=[\"x\", \"y\"])\n", + "pca_df.sample(50)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Let's plot these points to see if it looks like there is some structure in our data. We'll use the [Altair](https://altair-viz.github.io) library, which is a declarative visualization library, meaning that the presentation of our data will depend on the data itself -- for example, we'll say to use the two elements of the vectors for *x* and *y* coordinates but to use whether a document is legitimate or spam to determine how to color the point.\n", + "\n", + "We'll start by using the [`concat` function in the Pandas library](https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.concat.html) to make a data frame consisting of the original data frame with the PCA vector for each row." + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": { + "scrolled": false + }, + "outputs": [], + "source": [ + "plot_data = pd.concat([data.reset_index(), pca_df], axis=1)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Our next step will be to set up Altair, tell it how to encode our data frame in a plot, by using the `.encode(...)` method to tell it which values to use for x and y coordinates, as well as which value to use to decide how to color points. Altair will restrict us to plotting 5,000 points (so that the generated chart will not overwhelm our browser), so we'll also make sure to sample a subset of the data (in this case, 1,000 points)." + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "source": [ + "import altair as alt\n", + "alt.renderers.enable('nteract')\n", + "\n", + "alt.Chart(plot_data.sample(1000)).encode(x=\"x\", y=\"y\", color=\"label\").mark_point().interactive()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "That plot in particular is interactive (note the call to `.interactive()` at the end of the command), which means that you can pan around by dragging with the mouse or zoom with the mouse wheel. Try it out!\n", + "\n", + "Notice that, for the most part, even our simple shingling approach has identified some structure in the data: there is a clear dividing line between legitimate and spam documents. (It's important to remember that we're only using the labels to color points after we've placed them -- the PCA transformation isn't taking labels into account when mapping the vectors to two dimensions.)\n", + "\n", + "The next approach we'll try is called t-distributed stochastic neighbor embedding, or t-SNE for short. t-SNE learns a mapping from high-dimensional points to low-dimensional points so that points that are similar in high-dimensional space are likely to be similar in low-dimensional space as well. t-SNE can sometimes identify structure that simpler techniques like PCA can't, but this power comes at a cost: it is much more expensive to compute than PCA and doesn't parallelize well. (t-SNE also works best for visualizing two-dimensional data when it is reducing from tens of dimensions rather than hundreds or thousands. So, in some cases, you'll want to use a fast technique like PCA to reduce your data to a few dozen dimensions before using t-SNE. We haven't done that in this notebook, though.)\n", + "\n", + "So we can finish this notebook quickly and get on to the rest of our material, we'll only use t-SNE to visualize a subset of our data. We've [declared a helper function called `sample_corresponding`](mlworkflows/util.py), which takes a sequence of arrays or data frames, generates a set of random indices, and returns collections with the elements corresponding to the selected indices from each array or data frame. So if we had the collections `[1, 2, 3, 4, 5]` and `[2, 4, 6, 8, 10]`, a call to `sample_corresponding` asking for two elements might return `[[1, 4], [2, 8]]`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import sklearn.manifold\n", + "from mlworkflows import util as mlwutil\n", + "\n", + "np.random.seed(0xc0ffee)\n", + "sdf, sa = mlwutil.sample_corresponding(800, data, a)\n", + "\n", + "tsne = sklearn.manifold.TSNE()\n", + "tsne_a = tsne.fit_transform(sa)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "tsne_plot_data = pd.concat([sdf.reset_index(), pd.DataFrame(tsne_a, columns=[\"x\", \"y\"])], axis=1)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The Altair library, which we introduced while looking at our PCA results, is easy to use. However, to avoid cluttering our notebooks in a common case, we've [introduced a helper function called `plot_points`](mlworkflows/plot.py) that will just take a data frame and a data encoding before generating an interactive Altair scatterplot. (For more complicated cases, we'll still want to use Altair directly.)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from mlworkflows import plot\n", + "\n", + "plot.plot_points(tsne_plot_data, x=\"x\", y=\"y\", color=\"label\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "In this notebook, you've learned about two ways to visualize multidimensional data in two dimensions, which helps you to evaluate whether or not a given feature engineering approach is revealing structure in your data. In [our next notebook](02-evaluating-models.ipynb), you'll learn how to evaluate models based on the predictions that they make." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.3" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/types/@msrvida-python-program-analysis/test/parser.test.d.ts b/types/@msrvida-python-program-analysis/test/parser.test.d.ts new file mode 100644 index 000000000000..cb0ff5c3b541 --- /dev/null +++ b/types/@msrvida-python-program-analysis/test/parser.test.d.ts @@ -0,0 +1 @@ +export {}; diff --git a/types/@msrvida-python-program-analysis/test/program-builder.test.d.ts b/types/@msrvida-python-program-analysis/test/program-builder.test.d.ts new file mode 100644 index 000000000000..cb0ff5c3b541 --- /dev/null +++ b/types/@msrvida-python-program-analysis/test/program-builder.test.d.ts @@ -0,0 +1 @@ +export {}; diff --git a/types/@msrvida-python-program-analysis/test/slice.test.d.ts b/types/@msrvida-python-program-analysis/test/slice.test.d.ts new file mode 100644 index 000000000000..cb0ff5c3b541 --- /dev/null +++ b/types/@msrvida-python-program-analysis/test/slice.test.d.ts @@ -0,0 +1 @@ +export {}; diff --git a/types/@msrvida-python-program-analysis/test/testcell.d.ts b/types/@msrvida-python-program-analysis/test/testcell.d.ts new file mode 100644 index 000000000000..5152c4830bad --- /dev/null +++ b/types/@msrvida-python-program-analysis/test/testcell.d.ts @@ -0,0 +1,10 @@ +import { Cell } from ".."; +export declare class TestCell implements Cell { + text: string; + executionCount: number; + hasError: boolean; + executionEventId: string; + persistentId: string; + constructor(text: string, executionCount: number, executionEventId?: string, persistentId?: string, hasError?: boolean); + deepCopy(): this; +} From 9c6bd2a36116f5494665018fa6f3dee9f3fbfaad Mon Sep 17 00:00:00 2001 From: Jim Griesmer Date: Tue, 28 Jan 2020 00:22:56 -0800 Subject: [PATCH 16/51] Ensure cellHashProvider can be found --- .../editor-integration/codeLensFactory.ts | 25 +++++-------------- .../datascience/gather/gatherListener.ts | 18 +++---------- .../interactive-common/interactiveBase.ts | 3 +-- .../interactiveWindowTypes.ts | 4 +-- .../datascience/jupyter/jupyterServer.ts | 13 +++------- src/client/datascience/serviceRegistry.ts | 1 - 6 files changed, 16 insertions(+), 48 deletions(-) diff --git a/src/client/datascience/editor-integration/codeLensFactory.ts b/src/client/datascience/editor-integration/codeLensFactory.ts index 17723ef9d919..0baac1f1f6ba 100644 --- a/src/client/datascience/editor-integration/codeLensFactory.ts +++ b/src/client/datascience/editor-integration/codeLensFactory.ts @@ -93,30 +93,17 @@ export class CodeLensFactory implements ICodeLensFactory, IInteractiveWindowList handler.bind(this)(args); } - private doInitCellHashProvider(payload: string): void { + private doInitCellHashProvider(payload: INotebook): void { this.initCellHashProvider(payload).ignoreErrors(); } - private async initCellHashProvider(notebookUri: string) { - const nbUri: Uri = Uri.parse(notebookUri); - if (!nbUri) { - return; - } - - // First get the active server + private async initCellHashProvider(notebook: INotebook) { const activeServer = await this.jupyterExecution.getServer(await this.interactiveWindowProvider.getNotebookOptions()); - let nb: INotebook | undefined; - // If that works, see if there's a matching notebook running - if (activeServer) { - nb = await activeServer.getNotebook(nbUri); - - // If we have an executing notebook, get its gather execution service. - if (nb) { - this.hashProvider = nb.getCellHashProvider(); - if (this.hashProvider) { - this.hashProvider.updated(this.hashesUpdated.bind(this)); - } + if (activeServer && notebook) { + this.hashProvider = notebook.getCellHashProvider(); + if (this.hashProvider) { + this.hashProvider.updated(this.hashesUpdated.bind(this)); } } } diff --git a/src/client/datascience/gather/gatherListener.ts b/src/client/datascience/gather/gatherListener.ts index 3ce0ddeaa704..43a7e31fbc4f 100644 --- a/src/client/datascience/gather/gatherListener.ts +++ b/src/client/datascience/gather/gatherListener.ts @@ -78,25 +78,15 @@ export class GatherListener implements IInteractiveWindowListener { } } - private doInitGather(payload: string): void { + private doInitGather(payload: INotebook): void { this.initGather(payload).ignoreErrors(); } - private async initGather(notebookUri: string) { - this.notebookUri = Uri.parse(notebookUri); - - // First get the active server + private async initGather(notebook: INotebook) { const activeServer = await this.jupyterExecution.getServer(await this.interactiveWindowProvider.getNotebookOptions()); - let nb: INotebook | undefined; - // If that works, see if there's a matching notebook running - if (activeServer) { - nb = await activeServer.getNotebook(this.notebookUri); - - // If we have an executing notebook, get its gather execution service. - if (nb) { - this.gatherProvider = nb.getGatherProvider(); - } + if (activeServer && notebook) { + this.gatherProvider = notebook.getGatherProvider(); } } diff --git a/src/client/datascience/interactive-common/interactiveBase.ts b/src/client/datascience/interactive-common/interactiveBase.ts index f9c0f90b5a20..6b1ed6b718d0 100644 --- a/src/client/datascience/interactive-common/interactiveBase.ts +++ b/src/client/datascience/interactive-common/interactiveBase.ts @@ -952,8 +952,7 @@ export abstract class InteractiveBase extends WebViewHost { if (this._notebook) { diff --git a/src/client/datascience/interactive-common/interactiveWindowTypes.ts b/src/client/datascience/interactive-common/interactiveWindowTypes.ts index e51fe52fce43..f6ddeaeff464 100644 --- a/src/client/datascience/interactive-common/interactiveWindowTypes.ts +++ b/src/client/datascience/interactive-common/interactiveWindowTypes.ts @@ -5,7 +5,7 @@ import * as monacoEditor from 'monaco-editor/esm/vs/editor/editor.api'; import { IServerState } from '../../../datascience-ui/interactive-common/mainState'; import { CssMessages, IGetCssRequest, IGetCssResponse, IGetMonacoThemeRequest } from '../messages'; import { IGetMonacoThemeResponse } from '../monacoMessages'; -import { ICell, IInteractiveWindowInfo, IJupyterVariable, IJupyterVariablesRequest, IJupyterVariablesResponse } from '../types'; +import { ICell, IInteractiveWindowInfo, IJupyterVariable, IJupyterVariablesRequest, IJupyterVariablesResponse, INotebook } from '../types'; export enum InteractiveWindowMessages { StartCell = 'start_cell', @@ -299,7 +299,7 @@ export class IInteractiveWindowMapping { public [InteractiveWindowMessages.UpdateCell]: ICell; public [InteractiveWindowMessages.GotoCodeCell]: IGotoCode; public [InteractiveWindowMessages.CopyCodeCell]: ICopyCode; - public [InteractiveWindowMessages.NotebookExecutionActivated]: string; + public [InteractiveWindowMessages.NotebookExecutionActivated]: INotebook; public [InteractiveWindowMessages.RestartKernel]: never | undefined; public [InteractiveWindowMessages.SelectKernel]: IServerState | undefined; public [InteractiveWindowMessages.SelectJupyterServer]: never | undefined; diff --git a/src/client/datascience/jupyter/jupyterServer.ts b/src/client/datascience/jupyter/jupyterServer.ts index 206bd9f6b084..a07903463864 100644 --- a/src/client/datascience/jupyter/jupyterServer.ts +++ b/src/client/datascience/jupyter/jupyterServer.ts @@ -11,16 +11,8 @@ import { IAsyncDisposableRegistry, IConfigurationService, IDisposableRegistry } import { createDeferred, Deferred } from '../../common/utils/async'; import * as localize from '../../common/utils/localize'; import { noop } from '../../common/utils/misc'; -import { - IConnection, - IJupyterSession, - IJupyterSessionManager, - IJupyterSessionManagerFactory, - INotebook, - INotebookServer, - INotebookServerLaunchInfo -} from '../types'; import { IServiceContainer } from '../../ioc/types'; +import { IConnection, IJupyterSession, IJupyterSessionManager, IJupyterSessionManagerFactory, INotebook, INotebookServer, INotebookServerLaunchInfo } from '../types'; // This code is based on the examples here: // https://www.npmjs.com/package/@jupyterlab/services @@ -187,7 +179,8 @@ export class JupyterServerBase implements INotebookServer { _disposableRegistry: IDisposableRegistry, _configService: IConfigurationService, _serviceContainer: IServiceContainer, - _cancelToken?: CancellationToken): Promise { + _cancelToken?: CancellationToken + ): Promise { throw new Error('You forgot to override createNotebookInstance'); } diff --git a/src/client/datascience/serviceRegistry.ts b/src/client/datascience/serviceRegistry.ts index 8fe729102803..3f08e20e4769 100644 --- a/src/client/datascience/serviceRegistry.ts +++ b/src/client/datascience/serviceRegistry.ts @@ -141,7 +141,6 @@ export function registerTypes(serviceManager: IServiceManager) { serviceManager.add(ICellHashProvider, CellHashProvider); serviceManager.add(ICellHashLogger, CellHashLogger); serviceManager.addBinding(ICellHashLogger, INotebookExecutionLogger); - serviceManager.addBinding(ICellHashProvider, IInteractiveWindowListener); serviceManager.addBinding(IJupyterDebugger, ICellHashListener); serviceManager.addSingleton(INotebookEditorProvider, NativeEditorProvider); From f77f29a6cd641984fd8e3acc712cee6a440daf87 Mon Sep 17 00:00:00 2001 From: Jim Griesmer Date: Tue, 28 Jan 2020 14:58:15 -0800 Subject: [PATCH 17/51] Revert back to uri for NBExecutionActivated --- .../editor-integration/codeLensFactory.ts | 26 ++++++++++++++----- .../datascience/gather/gatherListener.ts | 18 ++++++++++--- .../interactive-common/interactiveBase.ts | 3 ++- .../interactiveWindowTypes.ts | 4 +-- src/client/datascience/serviceRegistry.ts | 2 +- .../datascience/dataScienceIocContainer.ts | 2 +- 6 files changed, 39 insertions(+), 16 deletions(-) diff --git a/src/client/datascience/editor-integration/codeLensFactory.ts b/src/client/datascience/editor-integration/codeLensFactory.ts index 0baac1f1f6ba..fd1fb1bf3f44 100644 --- a/src/client/datascience/editor-integration/codeLensFactory.ts +++ b/src/client/datascience/editor-integration/codeLensFactory.ts @@ -93,21 +93,33 @@ export class CodeLensFactory implements ICodeLensFactory, IInteractiveWindowList handler.bind(this)(args); } - private doInitCellHashProvider(payload: INotebook): void { + private doInitCellHashProvider(payload: string): void { this.initCellHashProvider(payload).ignoreErrors(); } - private async initCellHashProvider(notebook: INotebook) { + private async initCellHashProvider(notebookUri: string) { + const nbUri: Uri = Uri.parse(notebookUri); + if (!nbUri) { + return; + } + + // First get the active server const activeServer = await this.jupyterExecution.getServer(await this.interactiveWindowProvider.getNotebookOptions()); - if (activeServer && notebook) { - this.hashProvider = notebook.getCellHashProvider(); - if (this.hashProvider) { - this.hashProvider.updated(this.hashesUpdated.bind(this)); + let nb: INotebook | undefined; + // If that works, see if there's a matching notebook running + if (activeServer) { + nb = await activeServer.getNotebook(nbUri); + + // If we have an executing notebook, get its gather execution service. + if (nb) { + this.hashProvider = nb.getCellHashProvider(); + if (this.hashProvider) { + this.hashProvider.updated(this.hashesUpdated.bind(this)); + } } } } - private enumerateCommands(): string[] { let fullCommandList: string[]; // Add our non-debug commands diff --git a/src/client/datascience/gather/gatherListener.ts b/src/client/datascience/gather/gatherListener.ts index 43a7e31fbc4f..3ce0ddeaa704 100644 --- a/src/client/datascience/gather/gatherListener.ts +++ b/src/client/datascience/gather/gatherListener.ts @@ -78,15 +78,25 @@ export class GatherListener implements IInteractiveWindowListener { } } - private doInitGather(payload: INotebook): void { + private doInitGather(payload: string): void { this.initGather(payload).ignoreErrors(); } - private async initGather(notebook: INotebook) { + private async initGather(notebookUri: string) { + this.notebookUri = Uri.parse(notebookUri); + + // First get the active server const activeServer = await this.jupyterExecution.getServer(await this.interactiveWindowProvider.getNotebookOptions()); - if (activeServer && notebook) { - this.gatherProvider = notebook.getGatherProvider(); + let nb: INotebook | undefined; + // If that works, see if there's a matching notebook running + if (activeServer) { + nb = await activeServer.getNotebook(this.notebookUri); + + // If we have an executing notebook, get its gather execution service. + if (nb) { + this.gatherProvider = nb.getGatherProvider(); + } } } diff --git a/src/client/datascience/interactive-common/interactiveBase.ts b/src/client/datascience/interactive-common/interactiveBase.ts index 6b1ed6b718d0..f9c0f90b5a20 100644 --- a/src/client/datascience/interactive-common/interactiveBase.ts +++ b/src/client/datascience/interactive-common/interactiveBase.ts @@ -952,7 +952,8 @@ export abstract class InteractiveBase extends WebViewHost { if (this._notebook) { diff --git a/src/client/datascience/interactive-common/interactiveWindowTypes.ts b/src/client/datascience/interactive-common/interactiveWindowTypes.ts index f6ddeaeff464..e51fe52fce43 100644 --- a/src/client/datascience/interactive-common/interactiveWindowTypes.ts +++ b/src/client/datascience/interactive-common/interactiveWindowTypes.ts @@ -5,7 +5,7 @@ import * as monacoEditor from 'monaco-editor/esm/vs/editor/editor.api'; import { IServerState } from '../../../datascience-ui/interactive-common/mainState'; import { CssMessages, IGetCssRequest, IGetCssResponse, IGetMonacoThemeRequest } from '../messages'; import { IGetMonacoThemeResponse } from '../monacoMessages'; -import { ICell, IInteractiveWindowInfo, IJupyterVariable, IJupyterVariablesRequest, IJupyterVariablesResponse, INotebook } from '../types'; +import { ICell, IInteractiveWindowInfo, IJupyterVariable, IJupyterVariablesRequest, IJupyterVariablesResponse } from '../types'; export enum InteractiveWindowMessages { StartCell = 'start_cell', @@ -299,7 +299,7 @@ export class IInteractiveWindowMapping { public [InteractiveWindowMessages.UpdateCell]: ICell; public [InteractiveWindowMessages.GotoCodeCell]: IGotoCode; public [InteractiveWindowMessages.CopyCodeCell]: ICopyCode; - public [InteractiveWindowMessages.NotebookExecutionActivated]: INotebook; + public [InteractiveWindowMessages.NotebookExecutionActivated]: string; public [InteractiveWindowMessages.RestartKernel]: never | undefined; public [InteractiveWindowMessages.SelectKernel]: IServerState | undefined; public [InteractiveWindowMessages.SelectJupyterServer]: never | undefined; diff --git a/src/client/datascience/serviceRegistry.ts b/src/client/datascience/serviceRegistry.ts index 3f08e20e4769..cefee47d5e3b 100644 --- a/src/client/datascience/serviceRegistry.ts +++ b/src/client/datascience/serviceRegistry.ts @@ -114,7 +114,7 @@ export function registerTypes(serviceManager: IServiceManager) { serviceManager.add(IInteractiveWindow, InteractiveWindow); serviceManager.add(INotebookExporter, JupyterExporter); serviceManager.add(INotebookImporter, JupyterImporter); - serviceManager.add(INotebookServer, JupyterServerFactory); + serviceManager.addSingleton(INotebookServer, JupyterServerFactory); serviceManager.addSingleton(ICodeCssGenerator, CodeCssGenerator); serviceManager.addSingleton(IJupyterPasswordConnect, JupyterPasswordConnect); serviceManager.addSingleton(IStatusProvider, StatusProvider); diff --git a/src/test/datascience/dataScienceIocContainer.ts b/src/test/datascience/dataScienceIocContainer.ts index ef1d9032c9e0..522428bc1e11 100644 --- a/src/test/datascience/dataScienceIocContainer.ts +++ b/src/test/datascience/dataScienceIocContainer.ts @@ -417,7 +417,7 @@ export class DataScienceIocContainer extends UnitTestIocContainer { this.serviceManager.add(INotebookExporter, JupyterExporter); this.serviceManager.addSingleton(ILiveShareApi, MockLiveShareApi); this.serviceManager.addSingleton(IExtensions, MockExtensions); - this.serviceManager.add(INotebookServer, JupyterServerFactory); + this.serviceManager.addSingleton(INotebookServer, JupyterServerFactory); this.serviceManager.add(IJupyterCommandFactory, JupyterCommandFactory); this.serviceManager.addSingleton(IThemeFinder, ThemeFinder); this.serviceManager.addSingleton(ICodeCssGenerator, CodeCssGenerator); From c86099092582f145f7c44d450ce1ad9f8b494437 Mon Sep 17 00:00:00 2001 From: Jim Griesmer Date: Wed, 29 Jan 2020 11:58:53 -0800 Subject: [PATCH 18/51] Update new jupyterServerWrapper --- .../datascience/jupyter/jupyterServerWrapper.ts | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/client/datascience/jupyter/jupyterServerWrapper.ts b/src/client/datascience/jupyter/jupyterServerWrapper.ts index 88c79effa363..effb5edd4c3c 100644 --- a/src/client/datascience/jupyter/jupyterServerWrapper.ts +++ b/src/client/datascience/jupyter/jupyterServerWrapper.ts @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. 'use strict'; -import { inject, injectable, multiInject, optional } from 'inversify'; +import { inject, injectable } from 'inversify'; import * as uuid from 'uuid/v4'; import { Uri } from 'vscode'; import { CancellationToken } from 'vscode-jsonrpc'; @@ -11,7 +11,8 @@ import '../../common/extensions'; import { IFileSystem } from '../../common/platform/types'; import { IAsyncDisposableRegistry, IConfigurationService, IDisposableRegistry } from '../../common/types'; import { IInterpreterService } from '../../interpreter/contracts'; -import { IConnection, IDataScience, IJupyterSessionManagerFactory, INotebook, INotebookExecutionLogger, INotebookServer, INotebookServerLaunchInfo } from '../types'; +import { IServiceContainer } from '../../ioc/types'; +import { IConnection, IDataScience, IJupyterSessionManagerFactory, INotebook, INotebookServer, INotebookServerLaunchInfo } from '../types'; import { GuestJupyterServer } from './liveshare/guestJupyterServer'; import { HostJupyterServer } from './liveshare/hostJupyterServer'; import { IRoleBasedObject, RoleBasedFactory } from './liveshare/roleBasedFactory'; @@ -29,7 +30,7 @@ type JupyterServerClassType = { configService: IConfigurationService, sessionManager: IJupyterSessionManagerFactory, workspaceService: IWorkspaceService, - loggers: INotebookExecutionLogger[], + serviceContainer: IServiceContainer, appShell: IApplicationShell, fs: IFileSystem, interpreterService: IInterpreterService @@ -54,7 +55,7 @@ export class JupyterServerWrapper implements INotebookServer, ILiveShareHasRole @inject(IConfigurationService) configService: IConfigurationService, @inject(IJupyterSessionManagerFactory) sessionManager: IJupyterSessionManagerFactory, @inject(IWorkspaceService) workspaceService: IWorkspaceService, - @multiInject(INotebookExecutionLogger) @optional() loggers: INotebookExecutionLogger[] | undefined, + @inject(IServiceContainer) serviceContainer: IServiceContainer, @inject(IApplicationShell) appShell: IApplicationShell, @inject(IFileSystem) fs: IFileSystem, @inject(IInterpreterService) interpreterService: IInterpreterService @@ -72,7 +73,7 @@ export class JupyterServerWrapper implements INotebookServer, ILiveShareHasRole configService, sessionManager, workspaceService, - loggers ? loggers : [], + serviceContainer, appShell, fs, interpreterService From 7bda33f12ee5a0fca61d997b13792db3c7d65476 Mon Sep 17 00:00:00 2001 From: Jim Griesmer Date: Wed, 29 Jan 2020 14:50:56 -0800 Subject: [PATCH 19/51] Workaround fix for screwed up cellhashes Being fixed by Ian in a separate branch. --- src/client/datascience/interactive-common/interactiveBase.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/client/datascience/interactive-common/interactiveBase.ts b/src/client/datascience/interactive-common/interactiveBase.ts index f9c0f90b5a20..7d5efc69c1bc 100644 --- a/src/client/datascience/interactive-common/interactiveBase.ts +++ b/src/client/datascience/interactive-common/interactiveBase.ts @@ -1006,7 +1006,7 @@ export abstract class InteractiveBase extends WebViewHost Date: Fri, 31 Jan 2020 17:13:34 -0800 Subject: [PATCH 20/51] Gracefully fail if python-pgm-analysis is absent --- src/client/datascience/serviceRegistry.ts | 33 ++++++++++++++----- .../interactive-common/mainState.ts | 1 - 2 files changed, 25 insertions(+), 9 deletions(-) diff --git a/src/client/datascience/serviceRegistry.ts b/src/client/datascience/serviceRegistry.ts index 9a91b3806f14..af6ae2746778 100644 --- a/src/client/datascience/serviceRegistry.ts +++ b/src/client/datascience/serviceRegistry.ts @@ -1,8 +1,10 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. 'use strict'; +import { ConfigurationTarget } from 'vscode'; import { IExtensionSingleActivationService } from '../activation/types'; import { IWorkspaceService } from '../common/application/types'; +import { noop } from '../common/utils/misc'; import { IServiceManager } from '../ioc/types'; import { Activation } from './activation'; import { CodeCssGenerator } from './codeCssGenerator'; @@ -180,13 +182,28 @@ export function registerTypes(serviceManager: IServiceManager) { serviceManager.addSingleton(IJupyterInterpreterDependencyManager, JupyterInterpreterSubCommandExecutionService); } - const gatherEnabled: boolean = false; //serviceManager.get('python.dataScience', undefined).getSettings().datascience.enableGather ? true : false; - if (gatherEnabled) { - // tslint:disable-next-line: no-require-imports - const gather = require('./gather/gather'); - - serviceManager.add(IGatherProvider, gather.GatherProvider); - serviceManager.add(IGatherLogger, GatherLogger); - serviceManager.addBinding(IGatherLogger, INotebookExecutionLogger); + if (cfg.get('enableGather', false)) { + try { + registerGatherTypes(serviceManager); + } catch (ex) { + cfg.update('enableGather', false, ConfigurationTarget.Global).then(() => { + noop(); + }); + cfg.update('enableGather', false, ConfigurationTarget.Workspace).then(() => { + noop(); + }); + cfg.update('enableGather', false, ConfigurationTarget.WorkspaceFolder).then(() => { + noop(); + }); + } } } + +export function registerGatherTypes(serviceManager: IServiceManager) { + // tslint:disable-next-line: no-require-imports + const gather = require('./gather/gather'); + + serviceManager.add(IGatherProvider, gather.GatherProvider); + serviceManager.add(IGatherLogger, GatherLogger); + serviceManager.addBinding(IGatherLogger, INotebookExecutionLogger); +} diff --git a/src/datascience-ui/interactive-common/mainState.ts b/src/datascience-ui/interactive-common/mainState.ts index f4e65a80b883..e0be72abba79 100644 --- a/src/datascience-ui/interactive-common/mainState.ts +++ b/src/datascience-ui/interactive-common/mainState.ts @@ -111,7 +111,6 @@ const darkStyle = ` // This function generates test state when running under a browser instead of inside of export function generateTestState(filePath: string = '', editable: boolean = false): IMainState { const defaultSettings = getDefaultSettings(); - defaultSettings.enableGather = true; return { cellVMs: generateTestVMs(filePath, editable), From 987340fb4c1833d7ce93b813a0eb45e16a37906a Mon Sep 17 00:00:00 2001 From: Jim Griesmer Date: Fri, 31 Jan 2020 17:17:09 -0800 Subject: [PATCH 21/51] Resolve obvious unit test problems. --- .../datascience/gather/gather.unit.test.ts | 34 +++++++++---------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/src/test/datascience/gather/gather.unit.test.ts b/src/test/datascience/gather/gather.unit.test.ts index 5b85b7559a3c..d8d74b71adb9 100644 --- a/src/test/datascience/gather/gather.unit.test.ts +++ b/src/test/datascience/gather/gather.unit.test.ts @@ -134,23 +134,23 @@ suite('DataScience code gathering unit tests', () => { pythonSettings.setup(p => p.datascience).returns(() => dataScienceSettings.object); configurationService.setup(c => c.getSettings(TypeMoq.It.isAny())).returns(() => pythonSettings.object); appShell.setup(a => a.showInformationMessage(TypeMoq.It.isAny(), TypeMoq.It.isAny())).returns(() => Promise.resolve('')); - // const GatherProvider = new GatherProvider(configurationService.object, appShell.object, disposableRegistry.object, commandManager.object); - // const gatherLogger = new GatherLogger(GatherProvider, configurationService.object); + const gatherProvider = new GatherProvider(configurationService.object, appShell.object, disposableRegistry.object, commandManager.object); + const gatherLogger = new GatherLogger(gatherProvider, configurationService.object); - // test('Logs a cell execution', async () => { - // let count = 0; - // for (const c of codeCells) { - // await gatherLogger.postExecute(c, false); - // count += 1; - // assert.equal(GatherProvider.executionSlicer.executionLog.length, count); - // } - // }); + test('Logs a cell execution', async () => { + let count = 0; + for (const c of codeCells) { + await gatherLogger.postExecute(c, false); + count += 1; + assert.equal(gatherProvider.executionSlicer.executionLog.length, count); + } + }); - // test('Gathers program slices for a cell', async () => { - // const defaultCellMarker = '# %%'; - // const cell: IVscCell = codeCells[codeCells.length - 1]; - // const program = GatherProvider.gatherCode(cell); - // const expectedProgram = `# This file contains only the code required to produce the results of the gathered cell.\n${defaultCellMarker}\nfrom bokeh.plotting import show, figure, output_notebook\n\n${defaultCellMarker}\nx = [1,2,3,4,5]\ny = [21,9,15,17,4]\n\n${defaultCellMarker}\np=figure(title='demo',x_axis_label='x',y_axis_label='y')\n\n${defaultCellMarker}\np.line(x,y,line_width=2)\n\n${defaultCellMarker}\nshow(p)\n`; - // assert.equal(program.trim(), expectedProgram.trim()); - // }); + test('Gathers program slices for a cell', async () => { + const defaultCellMarker = '# %%'; + const cell: IVscCell = codeCells[codeCells.length - 1]; + const program = gatherProvider.gatherCode(cell); + const expectedProgram = `# This file contains only the code required to produce the results of the gathered cell.\n${defaultCellMarker}\nfrom bokeh.plotting import show, figure, output_notebook\n\n${defaultCellMarker}\nx = [1,2,3,4,5]\ny = [21,9,15,17,4]\n\n${defaultCellMarker}\np=figure(title='demo',x_axis_label='x',y_axis_label='y')\n\n${defaultCellMarker}\np.line(x,y,line_width=2)\n\n${defaultCellMarker}\nshow(p)\n`; + assert.equal(program.trim(), expectedProgram.trim()); + }); }); From c70c3c09a367b164691a923507e5c25b7808911c Mon Sep 17 00:00:00 2001 From: Jim Griesmer Date: Fri, 31 Jan 2020 17:42:07 -0800 Subject: [PATCH 22/51] Fixes to make sure that gather.ts is loadable. --- src/client/datascience/gather/gather.ts | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/src/client/datascience/gather/gather.ts b/src/client/datascience/gather/gather.ts index 494022e07353..0068602e5675 100644 --- a/src/client/datascience/gather/gather.ts +++ b/src/client/datascience/gather/gather.ts @@ -1,6 +1,6 @@ +import * as ppa from '@msrvida/python-program-analysis'; import { inject, injectable } from 'inversify'; import * as uuid from 'uuid/v4'; -import { Cell, CellSlice, DataflowAnalyzer, ExecutionLogSlicer } from '../../../../types/@msrvida-python-program-analysis'; import { IApplicationShell, ICommandManager } from '../../common/application/types'; import { traceInfo } from '../../common/logger'; import { IConfigurationService, IDisposableRegistry } from '../../common/types'; @@ -10,16 +10,13 @@ import { Common } from '../../common/utils/localize'; import { Identifiers } from '../constants'; import { CellState, ICell as IVscCell, IGatherProvider } from '../types'; -// tslint:disable-next-line: no-var-requires no-require-imports -const programAnalysis: typeof import('@msrvida-python-program-analysis') = require('@msrvida-python-program-analysis'); - /** * An adapter class to wrap the code gathering functionality from [microsoft/python-program-analysis](https://www.npmjs.com/package/@msrvida/python-program-analysis). */ @injectable() export class GatherProvider implements IGatherProvider { - private _executionSlicer: ExecutionLogSlicer; - private dataflowAnalyzer: DataflowAnalyzer; + private _executionSlicer: ppa.ExecutionLogSlicer; + private dataflowAnalyzer: ppa.DataflowAnalyzer; private _enabled: boolean; constructor( @@ -30,8 +27,8 @@ export class GatherProvider implements IGatherProvider { ) { this._enabled = this.configService.getSettings().datascience.enableGather ? true : false; - this.dataflowAnalyzer = new programAnalysis.DataflowAnalyzer(); - this._executionSlicer = new programAnalysis.ExecutionLogSlicer(this.dataflowAnalyzer); + this.dataflowAnalyzer = new ppa.DataflowAnalyzer(); + this._executionSlicer = new ppa.ExecutionLogSlicer(this.dataflowAnalyzer); if (this._enabled) { this.disposables.push(this.configService.getSettings().onDidChange(e => this.updateEnableGather(e))); @@ -101,7 +98,7 @@ export class GatherProvider implements IGatherProvider { /** * Accumulator to concatenate cell slices for a sliced program, preserving cell structures. */ -function concat(existingText: string, newText: CellSlice): string { +function concat(existingText: string, newText: ppa.CellSlice): string { // Include our cell marker so that cell slices are preserved return `${existingText}#%%\n${newText.textSliceLines}\n`; } @@ -110,10 +107,10 @@ function concat(existingText: string, newText: CellSlice): string { * This is called to convert VS Code ICells to Gather ICells for logging. * @param cell A cell object conforming to the VS Code cell interface */ -function convertVscToGatherCell(cell: IVscCell): Cell | undefined { +function convertVscToGatherCell(cell: IVscCell): ppa.Cell | undefined { // This should always be true since we only want to log code cells. Putting this here so types match for outputs property if (cell.data.cell_type === 'code') { - const result: Cell = { + const result: ppa.Cell = { // tslint:disable-next-line no-unnecessary-local-variable text: cell.data.source, From faf985f61b7aafc7a134ce80dfeeae5a02d7cc4f Mon Sep 17 00:00:00 2001 From: Jim Griesmer Date: Thu, 27 Feb 2020 15:10:52 -0800 Subject: [PATCH 23/51] Optional gather build, editing, running works. --- build/ci/templates/steps/initialization.yml | 12 +++- src/client/datascience/gather/gather.ts | 59 ++++++++++++------- src/client/datascience/serviceRegistry.ts | 18 +----- .../datascience/gather/gather.unit.test.ts | 36 ++++++----- 4 files changed, 69 insertions(+), 56 deletions(-) diff --git a/build/ci/templates/steps/initialization.yml b/build/ci/templates/steps/initialization.yml index d262eada79ed..6a99b43aadea 100644 --- a/build/ci/templates/steps/initialization.yml +++ b/build/ci/templates/steps/initialization.yml @@ -18,6 +18,7 @@ parameters: compile: 'true' sqlite: 'false' installVSCEorNPX: 'true' + enableGather: 'false' steps: - bash: | @@ -92,6 +93,15 @@ steps: verbose: true customCommand: ci + - task: Npm@1 + displayName: "Install optional python-program-analysis" + condition: and(succeeded(), eq('${{ parameters.enableGather }}', 'true')) + inputs: + workingDir: ${{ parameters.workingDirectory }} + command: custom + verbose: true + customCommand: "install @msrvida/python-program-analysis" + # On Mac, the command `node` doesn't always point to the current node version. # Debugger tests use the variable process.env.NODE_PATH - script: | @@ -101,7 +111,7 @@ steps: displayName: "Setup NODE_PATH for extension (Debugger Tests)" condition: and(succeeded(), eq(variables['agent.os'], 'Darwin')) - # Install vsce + # Install vsce - bash: | npm install -g vsce displayName: "Install vsce" diff --git a/src/client/datascience/gather/gather.ts b/src/client/datascience/gather/gather.ts index 6ed67d7e8f1d..0b55f477cfc8 100644 --- a/src/client/datascience/gather/gather.ts +++ b/src/client/datascience/gather/gather.ts @@ -1,4 +1,4 @@ -import * as ppa from '@msrvida/python-program-analysis'; +import * as ppatypes from '@msrvida-python-program-analysis'; import { inject, injectable } from 'inversify'; import * as uuid from 'uuid/v4'; import { IApplicationShell, ICommandManager } from '../../common/application/types'; @@ -15,8 +15,8 @@ import { CellState, ICell as IVscCell, IGatherProvider } from '../types'; */ @injectable() export class GatherProvider implements IGatherProvider { - private _executionSlicer: ppa.ExecutionLogSlicer; - private dataflowAnalyzer: ppa.DataflowAnalyzer; + private _executionSlicer: ppatypes.ExecutionLogSlicer | undefined; + private dataflowAnalyzer: ppatypes.DataflowAnalyzer | undefined; private _enabled: boolean; constructor( @@ -27,27 +27,38 @@ export class GatherProvider implements IGatherProvider { ) { this._enabled = this.configService.getSettings().datascience.enableGather ? true : false; - this.dataflowAnalyzer = new ppa.DataflowAnalyzer(); - this._executionSlicer = new ppa.ExecutionLogSlicer(this.dataflowAnalyzer); + try { + // tslint:disable-next-line: no-require-imports + const ppa = require('@msrvida/python-program-analysis') as typeof import('@msrvida-python-program-analysis'); - if (this._enabled) { - this.disposables.push( - this.configService.getSettings(undefined).onDidChange(e => this.updateEnableGather(e)) - ); - } + this.dataflowAnalyzer = new ppa.DataflowAnalyzer(); + this._executionSlicer = new ppa.ExecutionLogSlicer(this.dataflowAnalyzer); - traceInfo('Gathering tools have been activated'); + if (this._enabled) { + this.disposables.push( + this.configService.getSettings(undefined).onDidChange(e => this.updateEnableGather(e)) + ); + } + traceInfo('Gathering tools have been activated'); + } catch (ex) { + traceInfo('Gathering tools could not be activated'); + } } + public logExecution(vscCell: IVscCell): void { const gatherCell = convertVscToGatherCell(vscCell); if (gatherCell) { - this._executionSlicer.logExecution(gatherCell); + if (this._executionSlicer) { + this._executionSlicer.logExecution(gatherCell); + } } } public async resetLog(): Promise { - this._executionSlicer.reset(); + if (this._executionSlicer) { + this._executionSlicer.reset(); + } } /** @@ -64,12 +75,16 @@ export class GatherProvider implements IGatherProvider { this.configService.getSettings().datascience.defaultCellMarker || Identifiers.DefaultCodeCellMarker; // Call internal slice method - const slice = this._executionSlicer.sliceLatestExecution(gatherCell.persistentId); - const program = slice.cellSlices.reduce(concat, '').replace(/#%%/g, defaultCellMarker); - - // Add a comment at the top of the file explaining what gather does - const descriptor = localize.DataScience.gatheredScriptDescription(); - return descriptor.concat(program); + if (this._executionSlicer) { + const slice = this._executionSlicer.sliceLatestExecution(gatherCell.persistentId); + const program = slice.cellSlices.reduce(concat, '').replace(/#%%/g, defaultCellMarker); + + // Add a comment at the top of the file explaining what gather does + const descriptor = localize.DataScience.gatheredScriptDescription(); + return descriptor.concat(program); + } else { + return 'Gather not available'; + } } public get executionSlicer() { @@ -104,7 +119,7 @@ export class GatherProvider implements IGatherProvider { /** * Accumulator to concatenate cell slices for a sliced program, preserving cell structures. */ -function concat(existingText: string, newText: ppa.CellSlice): string { +function concat(existingText: string, newText: ppatypes.CellSlice): string { // Include our cell marker so that cell slices are preserved return `${existingText}#%%\n${newText.textSliceLines}\n`; } @@ -113,10 +128,10 @@ function concat(existingText: string, newText: ppa.CellSlice): string { * This is called to convert VS Code ICells to Gather ICells for logging. * @param cell A cell object conforming to the VS Code cell interface */ -function convertVscToGatherCell(cell: IVscCell): ppa.Cell | undefined { +function convertVscToGatherCell(cell: IVscCell): ppatypes.Cell | undefined { // This should always be true since we only want to log code cells. Putting this here so types match for outputs property if (cell.data.cell_type === 'code') { - const result: ppa.Cell = { + const result: ppatypes.Cell = { // tslint:disable-next-line no-unnecessary-local-variable text: cell.data.source, diff --git a/src/client/datascience/serviceRegistry.ts b/src/client/datascience/serviceRegistry.ts index 5db4e0c73029..21c57662262d 100644 --- a/src/client/datascience/serviceRegistry.ts +++ b/src/client/datascience/serviceRegistry.ts @@ -1,11 +1,9 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. 'use strict'; -import { ConfigurationTarget } from 'vscode'; import { IExtensionSingleActivationService } from '../activation/types'; import { IApplicationEnvironment, IWorkspaceService } from '../common/application/types'; import { UseCustomEditorApi } from '../common/constants'; -import { noop } from '../common/utils/misc'; import { IServiceManager } from '../ioc/types'; import { Activation } from './activation'; import { CodeCssGenerator } from './codeCssGenerator'; @@ -244,21 +242,7 @@ export function registerTypes(serviceManager: IServiceManager) { ); } - if (cfg.get('enableGather', false)) { - try { - registerGatherTypes(serviceManager); - } catch (ex) { - cfg.update('enableGather', false, ConfigurationTarget.Global).then(() => { - noop(); - }); - cfg.update('enableGather', false, ConfigurationTarget.Workspace).then(() => { - noop(); - }); - cfg.update('enableGather', false, ConfigurationTarget.WorkspaceFolder).then(() => { - noop(); - }); - } - } + registerGatherTypes(serviceManager); } export function registerGatherTypes(serviceManager: IServiceManager) { diff --git a/src/test/datascience/gather/gather.unit.test.ts b/src/test/datascience/gather/gather.unit.test.ts index 824849962392..540e86a213d3 100644 --- a/src/test/datascience/gather/gather.unit.test.ts +++ b/src/test/datascience/gather/gather.unit.test.ts @@ -147,22 +147,26 @@ suite('DataScience code gathering unit tests', () => { disposableRegistry.object, commandManager.object ); - const gatherLogger = new GatherLogger(gatherProvider, configurationService.object); - test('Logs a cell execution', async () => { - let count = 0; - for (const c of codeCells) { - await gatherLogger.postExecute(c, false); - count += 1; - assert.equal(gatherProvider.executionSlicer.executionLog.length, count); - } - }); + if (gatherProvider) { + const gatherLogger = new GatherLogger(gatherProvider, configurationService.object); + + test('Logs a cell execution', async () => { + let count = 0; + for (const c of codeCells) { + await gatherLogger.postExecute(c, false); + count += 1; + const logLength = gatherProvider.executionSlicer?.executionLog.length; + assert.equal(logLength, count); + } + }); - test('Gathers program slices for a cell', async () => { - const defaultCellMarker = '# %%'; - const cell: IVscCell = codeCells[codeCells.length - 1]; - const program = gatherProvider.gatherCode(cell); - const expectedProgram = `# This file contains only the code required to produce the results of the gathered cell.\n${defaultCellMarker}\nfrom bokeh.plotting import show, figure, output_notebook\n\n${defaultCellMarker}\nx = [1,2,3,4,5]\ny = [21,9,15,17,4]\n\n${defaultCellMarker}\np=figure(title='demo',x_axis_label='x',y_axis_label='y')\n\n${defaultCellMarker}\np.line(x,y,line_width=2)\n\n${defaultCellMarker}\nshow(p)\n`; - assert.equal(program.trim(), expectedProgram.trim()); - }); + test('Gathers program slices for a cell', async () => { + const defaultCellMarker = '# %%'; + const cell: IVscCell = codeCells[codeCells.length - 1]; + const program = gatherProvider.gatherCode(cell); + const expectedProgram = `# This file contains only the code required to produce the results of the gathered cell.\n${defaultCellMarker}\nfrom bokeh.plotting import show, figure, output_notebook\n\n${defaultCellMarker}\nx = [1,2,3,4,5]\ny = [21,9,15,17,4]\n\n${defaultCellMarker}\np=figure(title='demo',x_axis_label='x',y_axis_label='y')\n\n${defaultCellMarker}\np.line(x,y,line_width=2)\n\n${defaultCellMarker}\nshow(p)\n`; + assert.equal(program.trim(), expectedProgram.trim()); + }); + } }); From ff2cedb6e06c9b0380fa10dc2fccb5f699bf56aa Mon Sep 17 00:00:00 2001 From: Jim Griesmer Date: Thu, 27 Feb 2020 15:46:52 -0800 Subject: [PATCH 24/51] Remove unecessary getAll of IGatherProviders. --- src/client/datascience/gather/gatherListener.ts | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/client/datascience/gather/gatherListener.ts b/src/client/datascience/gather/gatherListener.ts index 48b0ae93b434..9bb000e04a5e 100644 --- a/src/client/datascience/gather/gatherListener.ts +++ b/src/client/datascience/gather/gatherListener.ts @@ -88,11 +88,9 @@ export class GatherListener implements IInteractiveWindowListener { } private doGather(payload: ICell): void { - if (this.serviceContainer.getAll(IGatherProvider)) { - this.gatherCodeInternal(payload).catch(err => { - this.applicationShell.showErrorMessage(err); - }); - } + this.gatherCodeInternal(payload).catch(err => { + this.applicationShell.showErrorMessage(err); + }); } private doInitGather(payload: string): void { From cbfe92e75a9d53719b323ce527318ca0bc7c372c Mon Sep 17 00:00:00 2001 From: Jim Griesmer Date: Thu, 27 Feb 2020 15:51:09 -0800 Subject: [PATCH 25/51] Code cleanup --- src/client/datascience/gather/gatherListener.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/client/datascience/gather/gatherListener.ts b/src/client/datascience/gather/gatherListener.ts index 9bb000e04a5e..a97c963c7654 100644 --- a/src/client/datascience/gather/gatherListener.ts +++ b/src/client/datascience/gather/gatherListener.ts @@ -8,7 +8,6 @@ import { IFileSystem } from '../../common/platform/types'; import { IConfigurationService } from '../../common/types'; import * as localize from '../../common/utils/localize'; import { noop } from '../../common/utils/misc'; -import { IServiceContainer } from '../../ioc/types'; import { generateCellsFromString } from '../cellFactory'; import { Identifiers } from '../constants'; import { IInteractiveWindowMapping, InteractiveWindowMessages } from '../interactive-common/interactiveWindowTypes'; @@ -42,8 +41,7 @@ export class GatherListener implements IInteractiveWindowListener { @inject(IJupyterExecution) private jupyterExecution: IJupyterExecution, @inject(IInteractiveWindowProvider) private interactiveWindowProvider: IInteractiveWindowProvider, @inject(IDocumentManager) private documentManager: IDocumentManager, - @inject(IFileSystem) private fileSystem: IFileSystem, - @inject(IServiceContainer) private serviceContainer: IServiceContainer + @inject(IFileSystem) private fileSystem: IFileSystem ) { } public dispose() { From d97ef68d23e5a961145524692d52be4a15c5ac13 Mon Sep 17 00:00:00 2001 From: Jim Griesmer Date: Fri, 28 Feb 2020 16:42:09 -0800 Subject: [PATCH 26/51] Fix gather notebook header - Also remove reference to python-program-analysis in package.json --- package-lock.json | 5 ----- package.json | 1 - package.nls.json | 2 +- src/client/common/utils/localize.ts | 2 +- src/client/datascience/gather/gather.ts | 20 ++++++++++---------- src/datascience-ui/common/index.ts | 10 +++++----- 6 files changed, 17 insertions(+), 23 deletions(-) diff --git a/package-lock.json b/package-lock.json index 53edb9653ffc..72e041b3eeb2 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1453,11 +1453,6 @@ "tinyqueue": "^1.1.0" } }, - "@msrvida/python-program-analysis": { - "version": "0.4.20", - "resolved": "https://pkgs.dev.azure.com/msresearch/_packaging/MSR-Python-Analysis/npm/registry/@msrvida/python-program-analysis/-/python-program-analysis-0.4.20.tgz", - "integrity": "sha1-NF+FuDLhWe7U5KkL48LDx8P1He8=" - }, "@nteract/markdown": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/@nteract/markdown/-/markdown-3.0.1.tgz", diff --git a/package.json b/package.json index 0f04a250a02e..45083ab38771 100644 --- a/package.json +++ b/package.json @@ -2864,7 +2864,6 @@ "@jupyterlab/services": "^4.2.0", "@koa/cors": "^3.0.0", "@loadable/component": "^5.12.0", - "@msrvida/python-program-analysis": "^0.4.20", "ansi-regex": "^4.1.0", "arch": "^2.1.0", "azure-storage": "^2.10.3", diff --git a/package.nls.json b/package.nls.json index c01143841338..ef08122e3dba 100644 --- a/package.nls.json +++ b/package.nls.json @@ -416,7 +416,7 @@ "DataScience.findJupyterCommandProgressCheckInterpreter": "Checking {0}.", "DataScience.findJupyterCommandProgressSearchCurrentPath": "Searching current path.", "DataScience.gatheredScriptDescription": "# This file contains only the code required to produce the results of the gathered cell.\n", - "DataScience.gatheredNotebookDescriptionInMarkdown": "# Gathered Notebook\nGenerated from ```{0}```\n\nThis notebook contains only the code and cells required to produce the same results as the gathered cell.\n\nPlease note that the python analysis is quite conservative, so if it is unsure whether a line of code is necessary for execution, it will err on the side of including it.", + "DataScience.gatheredNotebookDescriptionInMarkdown": "## Gathered Notebook\nGenerated from ```{0}```\n\nThis notebook contains only the code and cells required to produce the same results as the gathered cell.\n\nPlease note that the python analysis is quite conservative, so if it is unsure whether a line of code is necessary for execution, it will err on the side of including it.", "DataScience.savePngTitle": "Save Image", "DataScience.jupyterSelectURIQuickPickTitle": "Pick how to connect to Jupyter", "DataScience.jupyterSelectURIQuickPickPlaceholder": "Choose an option", diff --git a/src/client/common/utils/localize.ts b/src/client/common/utils/localize.ts index 37d882cd8e7c..e55861eebd01 100644 --- a/src/client/common/utils/localize.ts +++ b/src/client/common/utils/localize.ts @@ -721,7 +721,7 @@ export namespace DataScience { ); export const gatheredNotebookDescriptionInMarkdown = localize( 'DataScience.gatheredNotebookDescriptionInMarkdown', - '# Gathered Notebook\nGenerated from ```{0}```\n\nThis notebook contains only the code and cells required to produce the same results as the gathered cell.\n\nPlease note that the python analysis is quite conservative, so if it is unsure whether a line of code is necessary for execution, it will err on the side of including it.' + '## Gathered Notebook\nGenerated from ```{0}```\n\nThis notebook contains only the code and cells required to produce the same results as the gathered cell.\n\nPlease note that the python analysis is quite conservative, so if it is unsure whether a line of code is necessary for execution, it will err on the side of including it.' ); export const savePngTitle = localize('DataScience.savePngTitle', 'Save Image'); export const fallbackToUseActiveInterpeterAsKernel = localize( diff --git a/src/client/datascience/gather/gather.ts b/src/client/datascience/gather/gather.ts index 0b55f477cfc8..2c5064d0a5c3 100644 --- a/src/client/datascience/gather/gather.ts +++ b/src/client/datascience/gather/gather.ts @@ -65,6 +65,10 @@ export class GatherProvider implements IGatherProvider { * For a given code cell, returns a string representing a program containing all the code it depends on. */ public gatherCode(vscCell: IVscCell): string { + if (!this._executionSlicer) { + return '# %% [markdown]\n## Gather not available'; + } + const gatherCell = convertVscToGatherCell(vscCell); if (!gatherCell) { return ''; @@ -75,16 +79,12 @@ export class GatherProvider implements IGatherProvider { this.configService.getSettings().datascience.defaultCellMarker || Identifiers.DefaultCodeCellMarker; // Call internal slice method - if (this._executionSlicer) { - const slice = this._executionSlicer.sliceLatestExecution(gatherCell.persistentId); - const program = slice.cellSlices.reduce(concat, '').replace(/#%%/g, defaultCellMarker); - - // Add a comment at the top of the file explaining what gather does - const descriptor = localize.DataScience.gatheredScriptDescription(); - return descriptor.concat(program); - } else { - return 'Gather not available'; - } + const slice = this._executionSlicer.sliceLatestExecution(gatherCell.persistentId); + const program = slice.cellSlices.reduce(concat, '').replace(/#%%/g, defaultCellMarker); + + // Add a comment at the top of the file explaining what gather does + const descriptor = localize.DataScience.gatheredScriptDescription(); + return descriptor.concat(program); } public get executionSlicer() { diff --git a/src/datascience-ui/common/index.ts b/src/datascience-ui/common/index.ts index 6a194cf402c8..727c1ac53877 100644 --- a/src/datascience-ui/common/index.ts +++ b/src/datascience-ui/common/index.ts @@ -157,7 +157,7 @@ export function appendLineFeed(arr: string[], modifier?: (s: string) => string) export function generateMarkdownFromCodeLines(lines: string[]) { // Generate markdown by stripping out the comments and markdown header - return appendLineFeed(extractComments(lines.slice(1))); + return appendLineFeed(extractComments(lines.slice(lines.length > 1 ? 1 : 0))); } // tslint:disable-next-line: cyclomatic-complexity @@ -176,13 +176,13 @@ export function parseForComments( const isMultilineComment = trim.startsWith(SingleQuoteMultiline) ? SingleQuoteMultiline : trim.startsWith(DoubleQuoteMultiline) - ? DoubleQuoteMultiline - : undefined; + ? DoubleQuoteMultiline + : undefined; const isMultilineQuote = trim.includes(SingleQuoteMultiline) ? SingleQuoteMultiline : trim.includes(DoubleQuoteMultiline) - ? DoubleQuoteMultiline - : undefined; + ? DoubleQuoteMultiline + : undefined; // Check for ending quotes of multiline string if (insideMultilineQuote) { From d8f6f18b3ef7ed2ce4b82628f40f0d0033c8f2cc Mon Sep 17 00:00:00 2001 From: Jim Griesmer Date: Fri, 28 Feb 2020 16:46:14 -0800 Subject: [PATCH 27/51] Gather icon not only on hover --- src/datascience-ui/native-editor/nativeCell.tsx | 1 - 1 file changed, 1 deletion(-) diff --git a/src/datascience-ui/native-editor/nativeCell.tsx b/src/datascience-ui/native-editor/nativeCell.tsx index dbf2db37f32d..cb02b27dc054 100644 --- a/src/datascience-ui/native-editor/nativeCell.tsx +++ b/src/datascience-ui/native-editor/nativeCell.tsx @@ -588,7 +588,6 @@ export class NativeCell extends React.Component { 'Gather the code required to generate this cell into a new notebook' )} hidden={gatherDisabled} - className="hover-cell-button" > Date: Fri, 28 Feb 2020 17:13:11 -0800 Subject: [PATCH 28/51] Enable gather only in insiders --- package.json | 4 ++-- src/client/datascience/gather/gather.ts | 6 +++++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/package.json b/package.json index 45083ab38771..535c47dfa54e 100644 --- a/package.json +++ b/package.json @@ -1829,13 +1829,13 @@ "python.dataScience.enableGather": { "type": "boolean", "default": false, - "description": "Enable code gather for executed cells. For a gathered cell, that cell and only the code it depends on will be exported to a new notebook.", + "description": "Enable experimental code gathering for executed cells. For a gathered cell, that cell and only the code it depends on will be exported to a new notebook.", "scope": "resource" }, "python.dataScience.gatherToScript": { "type": "boolean", "default": true, - "description": "Gather code to a python script rather than a notebook.", + "description": "If experimental code gather is enabled, gather code to a python script rather than a notebook.", "scope": "resource" }, "python.dataScience.codeLenses": { diff --git a/src/client/datascience/gather/gather.ts b/src/client/datascience/gather/gather.ts index 2c5064d0a5c3..2139188e220a 100644 --- a/src/client/datascience/gather/gather.ts +++ b/src/client/datascience/gather/gather.ts @@ -25,7 +25,11 @@ export class GatherProvider implements IGatherProvider { @inject(IDisposableRegistry) private disposables: IDisposableRegistry, @inject(ICommandManager) private commandManager: ICommandManager ) { - this._enabled = this.configService.getSettings().datascience.enableGather ? true : false; + this._enabled = + this.configService.getSettings().datascience.enableGather && + this.configService.getSettings().insidersChannel !== 'off' + ? true + : false; try { // tslint:disable-next-line: no-require-imports From 99aeb0b292801e04a4b7063df0763fb26cee860d Mon Sep 17 00:00:00 2001 From: Jim Griesmer Date: Fri, 28 Feb 2020 17:14:49 -0800 Subject: [PATCH 29/51] Init gather only if enabled. --- src/client/datascience/gather/gather.ts | 30 +++++++++++++------------ 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/src/client/datascience/gather/gather.ts b/src/client/datascience/gather/gather.ts index 2139188e220a..951fb88937b5 100644 --- a/src/client/datascience/gather/gather.ts +++ b/src/client/datascience/gather/gather.ts @@ -31,21 +31,23 @@ export class GatherProvider implements IGatherProvider { ? true : false; - try { - // tslint:disable-next-line: no-require-imports - const ppa = require('@msrvida/python-program-analysis') as typeof import('@msrvida-python-program-analysis'); - - this.dataflowAnalyzer = new ppa.DataflowAnalyzer(); - this._executionSlicer = new ppa.ExecutionLogSlicer(this.dataflowAnalyzer); - - if (this._enabled) { - this.disposables.push( - this.configService.getSettings(undefined).onDidChange(e => this.updateEnableGather(e)) - ); + if (this._enabled) { + try { + // tslint:disable-next-line: no-require-imports + const ppa = require('@msrvida/python-program-analysis') as typeof import('@msrvida-python-program-analysis'); + + this.dataflowAnalyzer = new ppa.DataflowAnalyzer(); + this._executionSlicer = new ppa.ExecutionLogSlicer(this.dataflowAnalyzer); + + if (this._enabled) { + this.disposables.push( + this.configService.getSettings(undefined).onDidChange(e => this.updateEnableGather(e)) + ); + } + traceInfo('Gathering tools have been activated'); + } catch (ex) { + traceInfo('Gathering tools could not be activated'); } - traceInfo('Gathering tools have been activated'); - } catch (ex) { - traceInfo('Gathering tools could not be activated'); } } From 3fab0e14e45718b6923d03eec53372eb6d9867ce Mon Sep 17 00:00:00 2001 From: Jim Griesmer Date: Mon, 2 Mar 2020 10:57:22 -0800 Subject: [PATCH 30/51] Add link to survey --- package.nls.json | 2 +- src/client/common/utils/localize.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/package.nls.json b/package.nls.json index ef08122e3dba..d569ab198b56 100644 --- a/package.nls.json +++ b/package.nls.json @@ -416,7 +416,7 @@ "DataScience.findJupyterCommandProgressCheckInterpreter": "Checking {0}.", "DataScience.findJupyterCommandProgressSearchCurrentPath": "Searching current path.", "DataScience.gatheredScriptDescription": "# This file contains only the code required to produce the results of the gathered cell.\n", - "DataScience.gatheredNotebookDescriptionInMarkdown": "## Gathered Notebook\nGenerated from ```{0}```\n\nThis notebook contains only the code and cells required to produce the same results as the gathered cell.\n\nPlease note that the python analysis is quite conservative, so if it is unsure whether a line of code is necessary for execution, it will err on the side of including it.", + "DataScience.gatheredNotebookDescriptionInMarkdown": "## Gathered Notebook\nGenerated from ```{0}```\n\nThis notebook contains only the code and cells required to produce the same results as the gathered cell.\n\nPlease note that the python analysis is quite conservative, so if it is unsure whether a line of code is necessary for execution, it will err on the side of including it.\n\nAs this is an experimental feature, please let us know how well Gather works for you at [https://aka.ms/gathersurvey](https://aka.ms/gathersurvey)", "DataScience.savePngTitle": "Save Image", "DataScience.jupyterSelectURIQuickPickTitle": "Pick how to connect to Jupyter", "DataScience.jupyterSelectURIQuickPickPlaceholder": "Choose an option", diff --git a/src/client/common/utils/localize.ts b/src/client/common/utils/localize.ts index e55861eebd01..abcc78534ce2 100644 --- a/src/client/common/utils/localize.ts +++ b/src/client/common/utils/localize.ts @@ -721,7 +721,7 @@ export namespace DataScience { ); export const gatheredNotebookDescriptionInMarkdown = localize( 'DataScience.gatheredNotebookDescriptionInMarkdown', - '## Gathered Notebook\nGenerated from ```{0}```\n\nThis notebook contains only the code and cells required to produce the same results as the gathered cell.\n\nPlease note that the python analysis is quite conservative, so if it is unsure whether a line of code is necessary for execution, it will err on the side of including it.' + '## Gathered Notebook\nGenerated from ```{0}```\n\nThis notebook contains only the code and cells required to produce the same results as the gathered cell.\n\nPlease note that the python analysis is quite conservative, so if it is unsure whether a line of code is necessary for execution, it will err on the side of including it.\n\nAs this is an experimental feature, please let us know how well Gather works for you at [https://aka.ms/gathersurvey](https://aka.ms/gathersurvey)' ); export const savePngTitle = localize('DataScience.savePngTitle', 'Save Image'); export const fallbackToUseActiveInterpeterAsKernel = localize( From 77e0b1f1299a7c09ed47547168080ed017faa0ba Mon Sep 17 00:00:00 2001 From: Jim Griesmer Date: Mon, 2 Mar 2020 11:37:14 -0800 Subject: [PATCH 31/51] Fix spacing in initialization.yml --- build/ci/templates/steps/initialization.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/build/ci/templates/steps/initialization.yml b/build/ci/templates/steps/initialization.yml index 6a99b43aadea..2a99ab22433d 100644 --- a/build/ci/templates/steps/initialization.yml +++ b/build/ci/templates/steps/initialization.yml @@ -94,13 +94,13 @@ steps: customCommand: ci - task: Npm@1 - displayName: "Install optional python-program-analysis" - condition: and(succeeded(), eq('${{ parameters.enableGather }}', 'true')) - inputs: - workingDir: ${{ parameters.workingDirectory }} - command: custom - verbose: true - customCommand: "install @msrvida/python-program-analysis" + displayName: "Install optional python-program-analysis" + condition: and(succeeded(), eq('${{ parameters.enableGather }}', 'true')) + inputs: + workingDir: ${{ parameters.workingDirectory }} + command: custom + verbose: true + customCommand: "install @msrvida/python-program-analysis" # On Mac, the command `node` doesn't always point to the current node version. # Debugger tests use the variable process.env.NODE_PATH From cef4a87d551b5de72059a259791fe147581b3c4a Mon Sep 17 00:00:00 2001 From: Jim Griesmer Date: Mon, 2 Mar 2020 11:51:49 -0800 Subject: [PATCH 32/51] Make linter and prettier happy --- .../editor-integration/cellhashLogger.ts | 2 +- .../editor-integration/codeLensFactory.ts | 2 +- .../datascience/gather/gatherListener.ts | 2 +- src/client/datascience/gather/gatherLogger.ts | 2 +- .../datascience/jupyter/jupyterNotebook.ts | 18 +++++++-------- .../jupyter/jupyterServerWrapper.ts | 4 ++-- .../jupyter/liveshare/hostJupyterServer.ts | 22 +++++++++---------- src/datascience-ui/common/index.ts | 8 +++---- .../datascience/dataScienceIocContainer.ts | 2 +- .../codewatcher.unit.test.ts | 14 ++++++++++-- 10 files changed, 43 insertions(+), 33 deletions(-) diff --git a/src/client/datascience/editor-integration/cellhashLogger.ts b/src/client/datascience/editor-integration/cellhashLogger.ts index a73f7a718c84..3d3fd799fd8f 100644 --- a/src/client/datascience/editor-integration/cellhashLogger.ts +++ b/src/client/datascience/editor-integration/cellhashLogger.ts @@ -13,7 +13,7 @@ import { CellHashProvider } from './cellhashprovider'; // hashes for cells. @injectable() export class CellHashLogger implements ICellHashLogger { - constructor(@inject(ICellHashProvider) private provider: ICellHashProvider) { } + constructor(@inject(ICellHashProvider) private provider: ICellHashProvider) {} public async preExecute(cell: ICell, silent: boolean): Promise { const providerObj: CellHashProvider = this.provider as CellHashProvider; diff --git a/src/client/datascience/editor-integration/codeLensFactory.ts b/src/client/datascience/editor-integration/codeLensFactory.ts index f5cfcb79d033..8b25187b2353 100644 --- a/src/client/datascience/editor-integration/codeLensFactory.ts +++ b/src/client/datascience/editor-integration/codeLensFactory.ts @@ -40,7 +40,7 @@ export class CodeLensFactory implements ICodeLensFactory, IInteractiveWindowList @inject(IInteractiveWindowProvider) private interactiveWindowProvider: IInteractiveWindowProvider, @inject(IJupyterExecution) private jupyterExecution: IJupyterExecution, @inject(IFileSystem) private fileSystem: IFileSystem - ) { } + ) {} public dispose(): void { noop(); diff --git a/src/client/datascience/gather/gatherListener.ts b/src/client/datascience/gather/gatherListener.ts index a97c963c7654..5a21c342bc96 100644 --- a/src/client/datascience/gather/gatherListener.ts +++ b/src/client/datascience/gather/gatherListener.ts @@ -42,7 +42,7 @@ export class GatherListener implements IInteractiveWindowListener { @inject(IInteractiveWindowProvider) private interactiveWindowProvider: IInteractiveWindowProvider, @inject(IDocumentManager) private documentManager: IDocumentManager, @inject(IFileSystem) private fileSystem: IFileSystem - ) { } + ) {} public dispose() { noop(); diff --git a/src/client/datascience/gather/gatherLogger.ts b/src/client/datascience/gather/gatherLogger.ts index 249cff9816d5..18ce7fc0cb0f 100644 --- a/src/client/datascience/gather/gatherLogger.ts +++ b/src/client/datascience/gather/gatherLogger.ts @@ -12,7 +12,7 @@ export class GatherLogger implements IGatherLogger { constructor( @inject(IGatherProvider) private gather: IGatherProvider, @inject(IConfigurationService) private configService: IConfigurationService - ) { } + ) {} public async preExecute(_vscCell: IVscCell, _silent: boolean): Promise { // This function is just implemented here for compliance with the INotebookExecutionLogger interface diff --git a/src/client/datascience/jupyter/jupyterNotebook.ts b/src/client/datascience/jupyter/jupyterNotebook.ts index 155581ffa975..14bfa04293f2 100644 --- a/src/client/datascience/jupyter/jupyterNotebook.ts +++ b/src/client/datascience/jupyter/jupyterNotebook.ts @@ -770,15 +770,15 @@ export class JupyterNotebookBase implements INotebook { const cellMatcher = new CellMatcher(this.configService.getSettings(this.resource).datascience); return this.session ? this.session.requestExecute( - { - // Remove the cell marker if we have one. - code: cellMatcher.stripFirstMarker(code), - stop_on_error: false, - allow_stdin: true, // Allow when silent too in case runStartupCommands asks for a password - store_history: !silent // Silent actually means don't output anything. Store_history is what affects execution_count - }, - true - ) + { + // Remove the cell marker if we have one. + code: cellMatcher.stripFirstMarker(code), + stop_on_error: false, + allow_stdin: true, // Allow when silent too in case runStartupCommands asks for a password + store_history: !silent // Silent actually means don't output anything. Store_history is what affects execution_count + }, + true + ) : undefined; } catch (exc) { // Any errors generating a request should just be logged. User can't do anything about it. diff --git a/src/client/datascience/jupyter/jupyterServerWrapper.ts b/src/client/datascience/jupyter/jupyterServerWrapper.ts index 3931194d4123..b40a0d0c5222 100644 --- a/src/client/datascience/jupyter/jupyterServerWrapper.ts +++ b/src/client/datascience/jupyter/jupyterServerWrapper.ts @@ -34,11 +34,11 @@ import { HostJupyterServer } from './liveshare/hostJupyterServer'; import { IRoleBasedObject, RoleBasedFactory } from './liveshare/roleBasedFactory'; import { ILiveShareHasRole } from './liveshare/types'; -interface IJupyterServerInterface extends IRoleBasedObject, INotebookServer { } +interface IJupyterServerInterface extends IRoleBasedObject, INotebookServer {} // tslint:disable:callable-types type JupyterServerClassType = { - new( + new ( liveShare: ILiveShareApi, dataScience: IDataScience, asyncRegistry: IAsyncDisposableRegistry, diff --git a/src/client/datascience/jupyter/liveshare/hostJupyterServer.ts b/src/client/datascience/jupyter/liveshare/hostJupyterServer.ts index 5a1555c86563..2b21b31dc1fc 100644 --- a/src/client/datascience/jupyter/liveshare/hostJupyterServer.ts +++ b/src/client/datascience/jupyter/liveshare/hostJupyterServer.ts @@ -271,18 +271,18 @@ export class HostJupyterServer extends LiveShareParticipantHost(JupyterServerBas if (notebookMetadata?.kernelspec || resourceInterpreter?.displayName !== launchInfo.interpreter?.displayName) { const kernelInfo = await (launchInfo.connectionInfo.localLaunch ? this.kernelSelector.getKernelForLocalConnection( - resource, - sessionManager, - notebookMetadata, - false, - cancelToken - ) + resource, + sessionManager, + notebookMetadata, + false, + cancelToken + ) : this.kernelSelector.getKernelForRemoteConnection( - resource, - sessionManager, - notebookMetadata, - cancelToken - )); + resource, + sessionManager, + notebookMetadata, + cancelToken + )); const kernelInfoToUse = kernelInfo?.kernelSpec || kernelInfo?.kernelModel; if (kernelInfoToUse) { diff --git a/src/datascience-ui/common/index.ts b/src/datascience-ui/common/index.ts index 727c1ac53877..6293da3f237d 100644 --- a/src/datascience-ui/common/index.ts +++ b/src/datascience-ui/common/index.ts @@ -176,13 +176,13 @@ export function parseForComments( const isMultilineComment = trim.startsWith(SingleQuoteMultiline) ? SingleQuoteMultiline : trim.startsWith(DoubleQuoteMultiline) - ? DoubleQuoteMultiline - : undefined; + ? DoubleQuoteMultiline + : undefined; const isMultilineQuote = trim.includes(SingleQuoteMultiline) ? SingleQuoteMultiline : trim.includes(DoubleQuoteMultiline) - ? DoubleQuoteMultiline - : undefined; + ? DoubleQuoteMultiline + : undefined; // Check for ending quotes of multiline string if (insideMultilineQuote) { diff --git a/src/test/datascience/dataScienceIocContainer.ts b/src/test/datascience/dataScienceIocContainer.ts index 06c9e0628214..4178fd233e73 100644 --- a/src/test/datascience/dataScienceIocContainer.ts +++ b/src/test/datascience/dataScienceIocContainer.ts @@ -1297,7 +1297,7 @@ export class DataScienceIocContainer extends UnitTestIocContainer { this.postMessageToWebPanel(msg); }, // tslint:disable-next-line:no-any no-empty - setState: (_msg: any) => { }, + setState: (_msg: any) => {}, // tslint:disable-next-line:no-any no-empty getState: () => { return {}; diff --git a/src/test/datascience/editor-integration/codewatcher.unit.test.ts b/src/test/datascience/editor-integration/codewatcher.unit.test.ts index 1b9ec02e33df..6f737d44cb07 100644 --- a/src/test/datascience/editor-integration/codewatcher.unit.test.ts +++ b/src/test/datascience/editor-integration/codewatcher.unit.test.ts @@ -104,7 +104,12 @@ suite('DataScience Code Watcher Unit Tests', () => { // Setup the file system fileSystem.setup(f => f.arePathsSame(TypeMoq.It.isAnyString(), TypeMoq.It.isAnyString())).returns(() => true); - const codeLensFactory = new CodeLensFactory(configService.object, interactiveWindowProvider.object, jupyterExecution.object, fileSystem.object); + const codeLensFactory = new CodeLensFactory( + configService.object, + interactiveWindowProvider.object, + jupyterExecution.object, + fileSystem.object + ); serviceContainer .setup(c => c.get(TypeMoq.It.isValue(ICodeWatcher))) .returns( @@ -143,7 +148,12 @@ suite('DataScience Code Watcher Unit Tests', () => { return Promise.resolve(); }); - const codeLens = new CodeLensFactory(configService.object, interactiveWindowProvider.object, jupyterExecution.object, fileSystem.object); + const codeLens = new CodeLensFactory( + configService.object, + interactiveWindowProvider.object, + jupyterExecution.object, + fileSystem.object + ); codeWatcher = new CodeWatcher( interactiveWindowProvider.object, From d95d4414fefe3538e5022bcd4754f139c016024f Mon Sep 17 00:00:00 2001 From: Jim Griesmer Date: Mon, 2 Mar 2020 13:53:17 -0800 Subject: [PATCH 33/51] Make webpack happy with regard to gather --- build/ci/templates/globals.yml | 1 + build/ci/templates/steps/build.yml | 2 ++ build/webpack/webpack.extension.config.js | 7 ++++++- 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/build/ci/templates/globals.yml b/build/ci/templates/globals.yml index 653f763266ac..95ec5952bfc5 100644 --- a/build/ci/templates/globals.yml +++ b/build/ci/templates/globals.yml @@ -9,3 +9,4 @@ variables: CI_BRANCH_NAME: ${Build.SourceBranchName} npm_config_cache: $(Pipeline.Workspace)/.npm vmImageMacOS: 'macOS-10.15' + ENABLE_GATHER: 'true' diff --git a/build/ci/templates/steps/build.yml b/build/ci/templates/steps/build.yml index 167d87350b0f..ad1e0511e2d2 100644 --- a/build/ci/templates/steps/build.yml +++ b/build/ci/templates/steps/build.yml @@ -46,6 +46,8 @@ steps: - bash: | npm run package +# env: +# ENABLE_GATHER: ${ENABLE_GATHER} displayName: 'Build VSIX' - task: CopyFiles@2 diff --git a/build/webpack/webpack.extension.config.js b/build/webpack/webpack.extension.config.js index e35dfe609e73..8dfc9bc6c25f 100644 --- a/build/webpack/webpack.extension.config.js +++ b/build/webpack/webpack.extension.config.js @@ -13,6 +13,9 @@ const configFileName = path.join(constants.ExtensionRootDir, 'tsconfig.extension const existingModulesInOutDir = common.getListOfExistingModulesInOutDir(); // tslint:disable-next-line:no-var-requires no-require-imports const FileManagerPlugin = require('filemanager-webpack-plugin'); +// If ENABLE_GATHER variable is defined, don't exclude the python-program-analysis pacakge. +// See externals, below. +const ppaPackageList = process.env.ENABLE_GATHER ? [] : ['@msrvida/python-program-analysis']; const config = { mode: 'production', target: 'node', @@ -56,7 +59,9 @@ const config = { { enforce: 'post', test: /linebreak[\/\\]src[\/\\]linebreaker.js/, loader: 'transform-loader?brfs' } ] }, - externals: ['vscode', 'commonjs', ...existingModulesInOutDir], + // Externals essentially keeps webpack from trying to package them. The ppaPackageList variable is set to + // non-empty if the build pipeline has been authenticated to install @msrvida/python-program-analysis. + externals: ['vscode', 'commonjs', ...ppaPackageList, ...existingModulesInOutDir], plugins: [ ...common.getDefaultPlugins('extension'), // Copy pdfkit bits after extension builds. webpack can't handle pdfkit. From 9760a1d07b3c9677411b4f21edac0591773b9884 Mon Sep 17 00:00:00 2001 From: Jim Griesmer Date: Mon, 2 Mar 2020 14:19:54 -0800 Subject: [PATCH 34/51] Don't define ENABLE_GATHER env var --- build/ci/templates/globals.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/build/ci/templates/globals.yml b/build/ci/templates/globals.yml index 95ec5952bfc5..653f763266ac 100644 --- a/build/ci/templates/globals.yml +++ b/build/ci/templates/globals.yml @@ -9,4 +9,3 @@ variables: CI_BRANCH_NAME: ${Build.SourceBranchName} npm_config_cache: $(Pipeline.Workspace)/.npm vmImageMacOS: 'macOS-10.15' - ENABLE_GATHER: 'true' From 97a1a98f0866b0b6ba741007e9dae0e15813d11d Mon Sep 17 00:00:00 2001 From: Jim Griesmer Date: Tue, 3 Mar 2020 16:48:32 -0800 Subject: [PATCH 35/51] Couple minor issues found in test --- src/client/datascience/editor-integration/codeLensFactory.ts | 2 +- src/client/datascience/gather/gatherListener.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/client/datascience/editor-integration/codeLensFactory.ts b/src/client/datascience/editor-integration/codeLensFactory.ts index 8b25187b2353..639c9520652a 100644 --- a/src/client/datascience/editor-integration/codeLensFactory.ts +++ b/src/client/datascience/editor-integration/codeLensFactory.ts @@ -137,7 +137,7 @@ export class CodeLensFactory implements ICodeLensFactory, IInteractiveWindowList if (activeServer) { nb = await activeServer.getNotebook(nbUri); - // If we have an executing notebook, get its gather execution service. + // If we have an executing notebook, get its cell hash provider service. if (nb) { this.hashProvider = nb.getCellHashProvider(); if (this.hashProvider) { diff --git a/src/client/datascience/gather/gatherListener.ts b/src/client/datascience/gather/gatherListener.ts index 5a21c342bc96..77119b78cd27 100644 --- a/src/client/datascience/gather/gatherListener.ts +++ b/src/client/datascience/gather/gatherListener.ts @@ -116,7 +116,7 @@ export class GatherListener implements IInteractiveWindowListener { } private gatherCodeInternal = async (cell: ICell) => { - const slicedProgram = this.gatherProvider ? this.gatherProvider.gatherCode(cell) : ''; + const slicedProgram = this.gatherProvider ? this.gatherProvider.gatherCode(cell) : 'Gather internal error'; if (this.configService.getSettings().datascience.gatherToScript) { await this.showFile(slicedProgram, cell.file); From 8ae73d9eaed86c182033b02aea1a2d4e63f76cd7 Mon Sep 17 00:00:00 2001 From: Jim Griesmer Date: Wed, 4 Mar 2020 00:16:45 -0800 Subject: [PATCH 36/51] Fix a few tests --- .../editor-integration/codeLensFactory.ts | 17 ++---------- .../datascience/dataScienceIocContainer.ts | 10 ++++--- .../gotocell.functional.test.ts | 26 +++++++++++++++---- .../datascience/gather/gather.unit.test.ts | 25 ++++++++---------- 4 files changed, 40 insertions(+), 38 deletions(-) diff --git a/src/client/datascience/editor-integration/codeLensFactory.ts b/src/client/datascience/editor-integration/codeLensFactory.ts index 639c9520652a..32e90b4622e1 100644 --- a/src/client/datascience/editor-integration/codeLensFactory.ts +++ b/src/client/datascience/editor-integration/codeLensFactory.ts @@ -55,7 +55,7 @@ export class CodeLensFactory implements ICodeLensFactory, IInteractiveWindowList public onMessage(message: string, payload?: any) { switch (message) { case InteractiveWindowMessages.NotebookExecutionActivated: - this.handleMessage(message, payload, this.doInitCellHashProvider); + this.initCellHashProvider(payload).ignoreErrors(); break; case InteractiveWindowMessages.FinishCell: @@ -107,20 +107,6 @@ export class CodeLensFactory implements ICodeLensFactory, IInteractiveWindowList return codeLenses; } - // tslint:disable:no-any - private handleMessage( - _message: T, - payload: any, - handler: (args: M[T]) => void - ) { - const args = payload as M[T]; - handler.bind(this)(args); - } - - private doInitCellHashProvider(payload: string): void { - this.initCellHashProvider(payload).ignoreErrors(); - } - private async initCellHashProvider(notebookUri: string) { const nbUri: Uri = Uri.parse(notebookUri); if (!nbUri) { @@ -146,6 +132,7 @@ export class CodeLensFactory implements ICodeLensFactory, IInteractiveWindowList } } } + private enumerateCommands(resource: Resource): string[] { let fullCommandList: string[]; // Add our non-debug commands diff --git a/src/test/datascience/dataScienceIocContainer.ts b/src/test/datascience/dataScienceIocContainer.ts index 4178fd233e73..a0ea4f8e8419 100644 --- a/src/test/datascience/dataScienceIocContainer.ts +++ b/src/test/datascience/dataScienceIocContainer.ts @@ -163,6 +163,7 @@ import { ActiveEditorContextService } from '../../client/datascience/context/act import { DataViewer } from '../../client/datascience/data-viewing/dataViewer'; import { DataViewerProvider } from '../../client/datascience/data-viewing/dataViewerProvider'; import { DebugLocationTrackerFactory } from '../../client/datascience/debugLocationTrackerFactory'; +import { CellHashLogger } from '../../client/datascience/editor-integration/cellhashLogger'; import { CellHashProvider } from '../../client/datascience/editor-integration/cellhashprovider'; import { CodeLensFactory } from '../../client/datascience/editor-integration/codeLensFactory'; import { DataScienceCodeLensProvider } from '../../client/datascience/editor-integration/codelensprovider'; @@ -209,6 +210,7 @@ import { StatusProvider } from '../../client/datascience/statusProvider'; import { ThemeFinder } from '../../client/datascience/themeFinder'; import { ICellHashListener, + ICellHashLogger, ICellHashProvider, ICodeCssGenerator, ICodeLensFactory, @@ -606,12 +608,12 @@ export class DataScienceIocContainer extends UnitTestIocContainer { this.serviceManager.add(IInteractiveWindowListener, IntellisenseProvider); this.serviceManager.add(IInteractiveWindowListener, AutoSaveService); + this.serviceManager.add(IInteractiveWindowListener, GatherListener); this.serviceManager.add(IProtocolParser, ProtocolParser); this.serviceManager.addSingleton(IDebugService, MockDebuggerService); - this.serviceManager.addSingleton(ICellHashProvider, CellHashProvider); - this.serviceManager.addBinding(ICellHashProvider, IInteractiveWindowListener); - this.serviceManager.add(IInteractiveWindowListener, GatherListener); - this.serviceManager.addBinding(ICellHashProvider, INotebookExecutionLogger); + this.serviceManager.add(ICellHashProvider, CellHashProvider); + this.serviceManager.add(ICellHashLogger, CellHashLogger); + this.serviceManager.addBinding(ICellHashLogger, INotebookExecutionLogger); this.serviceManager.addBinding(IJupyterDebugger, ICellHashListener); this.serviceManager.add(IGatherProvider, GatherProvider); this.serviceManager.addSingleton(ICodeLensFactory, CodeLensFactory); diff --git a/src/test/datascience/editor-integration/gotocell.functional.test.ts b/src/test/datascience/editor-integration/gotocell.functional.test.ts index 56acb1b96bbb..ad54c2e35f00 100644 --- a/src/test/datascience/editor-integration/gotocell.functional.test.ts +++ b/src/test/datascience/editor-integration/gotocell.functional.test.ts @@ -82,19 +82,35 @@ suite('DataScience gotocell tests', () => { ): Promise { // Catch exceptions. Throw a specific assertion if the promise fails try { - const testDir = path.join(EXTENSION_ROOT_DIR, 'src', 'test', 'datascience'); + //const testDir = path.join(EXTENSION_ROOT_DIR, 'src', 'test', 'datascience'); + // tslint:disable-next-line: no-invalid-template-strings + const testDir = '${fileDirname}'; const server = await jupyterExecution.connectToNotebookServer({ usingDarkTheme, useDefaultConfig, workingDir: testDir, - purpose: purpose ? purpose : '1' + purpose: purpose ? purpose : Identifiers.HistoryPurpose, + enableDebugging: true }); if (expectFailure) { assert.ok(false, `Expected server to not be created`); } - return server - ? await server.createNotebook(undefined, Uri.parse(Identifiers.InteractiveWindowIdentity)) - : undefined; + + if (!server) { + return undefined; + } else { + const nb: INotebook = await server.createNotebook( + undefined, + Uri.parse(Identifiers.InteractiveWindowIdentity) + ); + nb.getCellHashProvider(); + const listener = (codeLensFactory as any) as IInteractiveWindowListener; + listener.onMessage( + InteractiveWindowMessages.NotebookExecutionActivated, + Identifiers.InteractiveWindowIdentity + ); + return nb; + } } catch (exc) { if (!expectFailure) { assert.ok(false, `Expected server to be created, but got ${exc}`); diff --git a/src/test/datascience/gather/gather.unit.test.ts b/src/test/datascience/gather/gather.unit.test.ts index 540e86a213d3..7b8ba2576a95 100644 --- a/src/test/datascience/gather/gather.unit.test.ts +++ b/src/test/datascience/gather/gather.unit.test.ts @@ -12,7 +12,6 @@ import { IPythonSettings } from '../../../client/common/types'; import { GatherProvider } from '../../../client/datascience/gather/gather'; -import { GatherLogger } from '../../../client/datascience/gather/gatherLogger'; import { ICell as IVscCell } from '../../../client/datascience/types'; // tslint:disable-next-line: max-func-body-length @@ -149,23 +148,21 @@ suite('DataScience code gathering unit tests', () => { ); if (gatherProvider) { - const gatherLogger = new GatherLogger(gatherProvider, configurationService.object); - - test('Logs a cell execution', async () => { - let count = 0; - for (const c of codeCells) { - await gatherLogger.postExecute(c, false); - count += 1; - const logLength = gatherProvider.executionSlicer?.executionLog.length; - assert.equal(logLength, count); - } - }); + // Disabling this test as by default gather cannot operate successfully without python-program-analysis. + // test('Logs a cell execution', async () => { + // let count = 0; + // for (const c of codeCells) { + // await gatherLogger.postExecute(c, false); + // count += 1; + // const logLength = gatherProvider.executionSlicer?.executionLog.length; + // assert.equal(logLength, count); + // } + // }); test('Gathers program slices for a cell', async () => { - const defaultCellMarker = '# %%'; const cell: IVscCell = codeCells[codeCells.length - 1]; const program = gatherProvider.gatherCode(cell); - const expectedProgram = `# This file contains only the code required to produce the results of the gathered cell.\n${defaultCellMarker}\nfrom bokeh.plotting import show, figure, output_notebook\n\n${defaultCellMarker}\nx = [1,2,3,4,5]\ny = [21,9,15,17,4]\n\n${defaultCellMarker}\np=figure(title='demo',x_axis_label='x',y_axis_label='y')\n\n${defaultCellMarker}\np.line(x,y,line_width=2)\n\n${defaultCellMarker}\nshow(p)\n`; + const expectedProgram = '# %% [markdown]\n## Gather not available'; assert.equal(program.trim(), expectedProgram.trim()); }); } From 8da69c9cadc2592ea6a5856568c87372fc4a0ea9 Mon Sep 17 00:00:00 2001 From: Jim Griesmer Date: Thu, 5 Mar 2020 16:35:42 -0800 Subject: [PATCH 37/51] Everything but dataScienceIocContainer.ts --- .prettierrc.js | 6 + .vscode/launch.json | 4 +- .../editor-integration/codeLensFactory.ts | 2 +- .../interactive-common/interactiveBase.ts | 5 +- src/client/datascience/serviceRegistry.ts | 172 +++++++----------- src/client/ioc/serviceManager.ts | 19 +- src/client/ioc/types.ts | 6 +- .../interactiveWindow.functional.test.tsx | 44 ++--- .../interactiveWindowTestHelpers.tsx | 10 +- .../nativeEditor.functional.test.tsx | 8 +- 10 files changed, 129 insertions(+), 147 deletions(-) diff --git a/.prettierrc.js b/.prettierrc.js index c24dc4c40c35..49f6747c18bc 100644 --- a/.prettierrc.js +++ b/.prettierrc.js @@ -8,6 +8,12 @@ module.exports = { options: { tabWidth: 2 } + }, + { + files: ['**/datascience/serviceRegistry.ts', '**/datascience/dataScienceIocContainer.ts'], + options: { + printWidth: 240 + } } ] }; diff --git a/.vscode/launch.json b/.vscode/launch.json index 39f2c4888690..aa0d3d008b10 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -185,7 +185,7 @@ "--ui=tdd", "--recursive", "--colors", - //"--grep", "", + //"--grep", "gathering", "--timeout=300000" ], "outFiles": [ @@ -234,7 +234,7 @@ "--ui=tdd", "--recursive", "--colors", - //"--grep", "", + "--grep", "Remove", "--timeout=300000" ], "env": { diff --git a/src/client/datascience/editor-integration/codeLensFactory.ts b/src/client/datascience/editor-integration/codeLensFactory.ts index 32e90b4622e1..4201f896f967 100644 --- a/src/client/datascience/editor-integration/codeLensFactory.ts +++ b/src/client/datascience/editor-integration/codeLensFactory.ts @@ -11,7 +11,7 @@ import * as localize from '../../common/utils/localize'; import { noop } from '../../common/utils/misc'; import { generateCellRangesFromDocument } from '../cellFactory'; import { CodeLensCommands, Commands } from '../constants'; -import { IInteractiveWindowMapping, InteractiveWindowMessages } from '../interactive-common/interactiveWindowTypes'; +import { InteractiveWindowMessages } from '../interactive-common/interactiveWindowTypes'; import { ICell, ICellHashProvider, diff --git a/src/client/datascience/interactive-common/interactiveBase.ts b/src/client/datascience/interactive-common/interactiveBase.ts index 55af6b73928e..1df1ae7bcaaf 100644 --- a/src/client/datascience/interactive-common/interactiveBase.ts +++ b/src/client/datascience/interactive-common/interactiveBase.ts @@ -581,6 +581,7 @@ export abstract class InteractiveBase extends WebViewHost { + (cells: ICell[]) => { // Combine the cell data with the possible input data (so we don't lose anything that might have already been in the cells) const combined = cells.map(this.combineData.bind(undefined, data)); @@ -596,7 +597,7 @@ export abstract class InteractiveBase extends WebViewHost c.state === CellState.error) === undefined; } }, diff --git a/src/client/datascience/serviceRegistry.ts b/src/client/datascience/serviceRegistry.ts index a56a787bdf3e..3e62cd01a7ec 100644 --- a/src/client/datascience/serviceRegistry.ts +++ b/src/client/datascience/serviceRegistry.ts @@ -114,135 +114,86 @@ import { IThemeFinder } from './types'; +// README: Did you make sure "dataScienceIocContainer.ts" has also been updated appropriately? + // tslint:disable-next-line: max-func-body-length export function registerTypes(serviceManager: IServiceManager) { - const useCustomEditorApi = serviceManager.get(IApplicationEnvironment).packageJson - .enableProposedApi; - serviceManager.addSingletonInstance(UseCustomEditorApi, useCustomEditorApi); + const useCustomEditorApi = serviceManager.get(IApplicationEnvironment).packageJson.enableProposedApi; - serviceManager.addSingleton( - IDataScienceCodeLensProvider, - DataScienceCodeLensProvider - ); - serviceManager.addSingleton(IDataScience, DataScience); - serviceManager.addSingleton(IJupyterExecution, JupyterExecutionFactory); - serviceManager.addSingleton( - IDataScienceCommandListener, - InteractiveWindowCommandListener - ); - serviceManager.addSingleton(IInteractiveWindowProvider, InteractiveWindowProvider); - serviceManager.add(IInteractiveWindow, InteractiveWindow); - serviceManager.add(INotebookExporter, JupyterExporter); - serviceManager.add(INotebookImporter, JupyterImporter); - serviceManager.add(INotebookServer, JupyterServerWrapper); - serviceManager.addSingleton(ICodeCssGenerator, CodeCssGenerator); - serviceManager.addSingleton(IJupyterPasswordConnect, JupyterPasswordConnect); - serviceManager.addSingleton(IStatusProvider, StatusProvider); - serviceManager.addSingleton( - IJupyterSessionManagerFactory, - JupyterSessionManagerFactory - ); - serviceManager.addSingleton(IJupyterVariables, JupyterVariables); + serviceManager.add(ICellHashLogger, CellHashLogger, undefined, [INotebookExecutionLogger]); + serviceManager.add(ICellHashProvider, CellHashProvider); serviceManager.add(ICodeWatcher, CodeWatcher); - serviceManager.add(IJupyterCommandFactory, JupyterCommandFactory); - serviceManager.addSingleton(IThemeFinder, ThemeFinder); - serviceManager.addSingleton(IDataViewerProvider, DataViewerProvider); + serviceManager.add(IDataScienceErrorHandler, DataScienceErrorHandler); serviceManager.add(IDataViewer, DataViewer); - serviceManager.addSingleton(IExtensionSingleActivationService, Decorator); - serviceManager.add(IInteractiveWindowListener, IntellisenseProvider); - serviceManager.add(IInteractiveWindowListener, LinkProvider); - serviceManager.add(IInteractiveWindowListener, ShowPlotListener); + serviceManager.add(IInteractiveWindow, InteractiveWindow); serviceManager.add(IInteractiveWindowListener, AutoSaveService); serviceManager.add(IInteractiveWindowListener, DebugListener); serviceManager.add(IInteractiveWindowListener, GatherListener); - serviceManager.addSingleton(IPlotViewerProvider, PlotViewerProvider); - serviceManager.add(IPlotViewer, PlotViewer); - serviceManager.addSingleton(IJupyterDebugger, JupyterDebugger); - serviceManager.add(IDataScienceErrorHandler, DataScienceErrorHandler); - serviceManager.addSingleton(ICodeLensFactory, CodeLensFactory); - serviceManager.add(ICellHashProvider, CellHashProvider); - serviceManager.add(ICellHashLogger, CellHashLogger); - serviceManager.addBinding(ICellHashLogger, INotebookExecutionLogger); - serviceManager.addSingleton(IInteractiveWindowListener, DataScienceSurveyBannerLogger); - - serviceManager.addBinding(IJupyterDebugger, ICellHashListener); - serviceManager.addSingleton( - INotebookEditorProvider, - useCustomEditorApi ? NativeEditorProvider : NativeEditorProviderOld - ); - serviceManager.add(INotebookStorage, NativeEditorStorage); + serviceManager.add(IInteractiveWindowListener, IntellisenseProvider); + serviceManager.add(IInteractiveWindowListener, LinkProvider); + serviceManager.add(IInteractiveWindowListener, ShowPlotListener); + serviceManager.add(IJupyterCommandFactory, JupyterCommandFactory); serviceManager.add(INotebookEditor, useCustomEditorApi ? NativeEditor : NativeEditorOldWebView); + serviceManager.add(INotebookExporter, JupyterExporter); + serviceManager.add(INotebookImporter, JupyterImporter); + serviceManager.add(INotebookServer, JupyterServerWrapper); + serviceManager.add(INotebookStorage, NativeEditorStorage); + serviceManager.add(IPlotViewer, PlotViewer); + serviceManager.addSingleton(ActiveEditorContextService, ActiveEditorContextService); + serviceManager.addSingleton(CommandRegistry, CommandRegistry); + serviceManager.addSingleton(DataViewerDependencyService, DataViewerDependencyService); + serviceManager.addSingleton(ICodeCssGenerator, CodeCssGenerator); + serviceManager.addSingleton(ICodeLensFactory, CodeLensFactory, undefined, [IInteractiveWindowListener]); + serviceManager.addSingleton(IDataScience, DataScience); + serviceManager.addSingleton(IDataScienceCodeLensProvider, DataScienceCodeLensProvider); + serviceManager.addSingleton(IDataScienceCommandListener, InteractiveWindowCommandListener); serviceManager.addSingleton(IDataScienceCommandListener, NativeEditorCommandListener); - serviceManager.addBinding(ICodeLensFactory, IInteractiveWindowListener); + serviceManager.addSingleton(IDataViewerProvider, DataViewerProvider); serviceManager.addSingleton(IDebugLocationTracker, DebugLocationTrackerFactory); - serviceManager.addSingleton(JupyterCommandFinder, JupyterCommandFinder); serviceManager.addSingleton(IExtensionSingleActivationService, Activation); - serviceManager.addSingleton(KernelService, KernelService); - serviceManager.addSingleton(NotebookStarter, NotebookStarter); - serviceManager.addSingleton(KernelSelector, KernelSelector); - serviceManager.addSingleton(KernelSelectionProvider, KernelSelectionProvider); - serviceManager.addSingleton(CommandRegistry, CommandRegistry); - serviceManager.addSingleton( - JupyterServerSelectorCommand, - JupyterServerSelectorCommand - ); - serviceManager.addSingleton(KernelSwitcherCommand, KernelSwitcherCommand); - serviceManager.addSingleton(KernelSwitcher, KernelSwitcher); - serviceManager.addSingleton(JupyterServerSelector, JupyterServerSelector); - serviceManager.addSingleton( - JupyterCommandLineSelectorCommand, - JupyterCommandLineSelectorCommand - ); + serviceManager.addSingleton(IExtensionSingleActivationService, Decorator); + serviceManager.addSingleton(IExtensionSingleActivationService, JupyterInterpreterSelectionCommand); + serviceManager.addSingleton(IExtensionSingleActivationService, PreWarmActivatedJupyterEnvironmentVariables); + serviceManager.addSingleton(IExtensionSingleActivationService, ServerPreload); + serviceManager.addSingleton(IExtensionSingleActivationService, ServerPreload); + serviceManager.addSingleton(IInteractiveWindowListener, DataScienceSurveyBannerLogger); + serviceManager.addSingleton(IInteractiveWindowProvider, InteractiveWindowProvider); + serviceManager.addSingleton(IJupyterDebugger, JupyterDebugger, undefined, [ICellHashListener]); + serviceManager.addSingleton(IJupyterExecution, JupyterExecutionFactory); + serviceManager.addSingleton(IJupyterPasswordConnect, JupyterPasswordConnect); + serviceManager.addSingleton(IJupyterSessionManagerFactory, JupyterSessionManagerFactory); + serviceManager.addSingleton(IJupyterVariables, JupyterVariables); + serviceManager.addSingleton(INotebookEditorProvider, useCustomEditorApi ? NativeEditorProvider : NativeEditorProviderOld); + serviceManager.addSingleton(IPlotViewerProvider, PlotViewerProvider); + serviceManager.addSingleton(IStatusProvider, StatusProvider); + serviceManager.addSingleton(IThemeFinder, ThemeFinder); + serviceManager.addSingleton(JupyterCommandFinder, JupyterCommandFinder); serviceManager.addSingleton(JupyterCommandLineSelector, JupyterCommandLineSelector); - serviceManager.addSingleton( - JupyterInterpreterStateStore, - JupyterInterpreterStateStore - ); - serviceManager.addSingleton( - IExtensionSingleActivationService, - JupyterInterpreterSelectionCommand - ); - serviceManager.addSingleton( - IExtensionSingleActivationService, - PreWarmActivatedJupyterEnvironmentVariables - ); + serviceManager.addSingleton(JupyterCommandLineSelectorCommand, JupyterCommandLineSelectorCommand); + serviceManager.addSingleton(JupyterInterpreterDependencyService, JupyterInterpreterDependencyService); + serviceManager.addSingleton(JupyterInterpreterOldCacheStateStore, JupyterInterpreterOldCacheStateStore); serviceManager.addSingleton(JupyterInterpreterSelector, JupyterInterpreterSelector); - serviceManager.addSingleton( - JupyterInterpreterDependencyService, - JupyterInterpreterDependencyService - ); serviceManager.addSingleton(JupyterInterpreterService, JupyterInterpreterService); - serviceManager.addSingleton( - JupyterInterpreterOldCacheStateStore, - JupyterInterpreterOldCacheStateStore - ); - serviceManager.addSingleton(ActiveEditorContextService, ActiveEditorContextService); + serviceManager.addSingleton(JupyterInterpreterStateStore, JupyterInterpreterStateStore); + serviceManager.addSingleton(JupyterServerSelector, JupyterServerSelector); + serviceManager.addSingleton(JupyterServerSelectorCommand, JupyterServerSelectorCommand); + serviceManager.addSingleton(KernelSelectionProvider, KernelSelectionProvider); + serviceManager.addSingleton(KernelSelector, KernelSelector); + serviceManager.addSingleton(KernelService, KernelService); + serviceManager.addSingleton(KernelSwitcher, KernelSwitcher); + serviceManager.addSingleton(KernelSwitcherCommand, KernelSwitcherCommand); + serviceManager.addSingleton(NotebookStarter, NotebookStarter); serviceManager.addSingleton(ProgressReporter, ProgressReporter); - serviceManager.addSingleton(IExtensionSingleActivationService, ServerPreload); - serviceManager.addSingleton(DataViewerDependencyService, DataViewerDependencyService); + serviceManager.addSingletonInstance(UseCustomEditorApi, useCustomEditorApi); // Temporary code, to allow users to revert to the old behavior. - const cfg = serviceManager - .get(IWorkspaceService) - .getConfiguration('python.dataScience', undefined); + const cfg = serviceManager.get(IWorkspaceService).getConfiguration('python.dataScience', undefined); if (cfg.get('useOldJupyterServer', false)) { - serviceManager.addSingleton( - IJupyterSubCommandExecutionService, - JupyterCommandFinderInterpreterExecutionService - ); - serviceManager.addSingleton( - IJupyterInterpreterDependencyManager, - JupyterCommandInterpreterDependencyService - ); + serviceManager.addSingleton(IJupyterInterpreterDependencyManager, JupyterCommandInterpreterDependencyService); + serviceManager.addSingleton(IJupyterSubCommandExecutionService, JupyterCommandFinderInterpreterExecutionService); } else { - serviceManager.addSingleton( - IJupyterSubCommandExecutionService, - JupyterInterpreterSubCommandExecutionService - ); - serviceManager.addSingleton( - IJupyterInterpreterDependencyManager, - JupyterInterpreterSubCommandExecutionService - ); + serviceManager.addSingleton(IJupyterInterpreterDependencyManager, JupyterInterpreterSubCommandExecutionService); + serviceManager.addSingleton(IJupyterSubCommandExecutionService, JupyterInterpreterSubCommandExecutionService); } registerGatherTypes(serviceManager); @@ -253,6 +204,5 @@ export function registerGatherTypes(serviceManager: IServiceManager) { const gather = require('./gather/gather'); serviceManager.add(IGatherProvider, gather.GatherProvider); - serviceManager.add(IGatherLogger, GatherLogger); - serviceManager.addBinding(IGatherLogger, INotebookExecutionLogger); + serviceManager.add(IGatherLogger, GatherLogger, undefined, [INotebookExecutionLogger]); } diff --git a/src/client/ioc/serviceManager.ts b/src/client/ioc/serviceManager.ts index 115f8dd7a4ed..b63559ab781d 100644 --- a/src/client/ioc/serviceManager.ts +++ b/src/client/ioc/serviceManager.ts @@ -13,7 +13,8 @@ export class ServiceManager implements IServiceManager { serviceIdentifier: identifier, // tslint:disable-next-line:no-any constructor: new (...args: any[]) => T, - name?: string | number | symbol | undefined + name?: string | number | symbol | undefined, + bindings?: symbol[] ): void { if (name) { this.container @@ -23,6 +24,12 @@ export class ServiceManager implements IServiceManager { } else { this.container.bind(serviceIdentifier).to(constructor); } + + if (bindings) { + bindings.forEach(binding => { + this.addBinding(serviceIdentifier, binding); + }); + } } public addFactory( factoryIdentifier: interfaces.ServiceIdentifier>, @@ -39,7 +46,8 @@ export class ServiceManager implements IServiceManager { serviceIdentifier: identifier, // tslint:disable-next-line:no-any constructor: new (...args: any[]) => T, - name?: string | number | symbol | undefined + name?: string | number | symbol | undefined, + bindings?: symbol[] ): void { if (name) { this.container @@ -53,7 +61,14 @@ export class ServiceManager implements IServiceManager { .to(constructor) .inSingletonScope(); } + + if (bindings) { + bindings.forEach(binding => { + this.addBinding(serviceIdentifier, binding); + }); + } } + public addSingletonInstance( serviceIdentifier: identifier, instance: T, diff --git a/src/client/ioc/types.ts b/src/client/ioc/types.ts index 02787d4a1ea0..31312453da75 100644 --- a/src/client/ioc/types.ts +++ b/src/client/ioc/types.ts @@ -29,12 +29,14 @@ export interface IServiceManager { add( serviceIdentifier: interfaces.ServiceIdentifier, constructor: ClassType, - name?: string | number | symbol + name?: string | number | symbol | undefined, + bindings?: symbol[] ): void; addSingleton( serviceIdentifier: interfaces.ServiceIdentifier, constructor: ClassType, - name?: string | number | symbol + name?: string | number | symbol, + bindings?: symbol[] ): void; addSingletonInstance( serviceIdentifier: interfaces.ServiceIdentifier, diff --git a/src/test/datascience/interactiveWindow.functional.test.tsx b/src/test/datascience/interactiveWindow.functional.test.tsx index 7fdb88007e0b..5cda6d02bc7c 100644 --- a/src/test/datascience/interactiveWindow.functional.test.tsx +++ b/src/test/datascience/interactiveWindow.functional.test.tsx @@ -336,32 +336,32 @@ for _ in range(50): sys.stdout.write('\\r')`; addMockData(ioc, badPanda, `pandas has no attribute 'read'`, 'text/html', 'error'); - addMockData(ioc, goodPanda, `A table`, 'text/html'); - addMockData(ioc, matPlotLib, matPlotLibResults, 'text/html'); - const cursors = ['|', '/', '-', '\\']; - let cursorPos = 0; - let loops = 3; - addContinuousMockData(ioc, spinningCursor, async _c => { - const result = `${cursors[cursorPos]}\r`; - cursorPos += 1; - if (cursorPos >= cursors.length) { - cursorPos = 0; - loops -= 1; - } - return Promise.resolve({ result: result, haveMore: loops > 0 }); - }); + // addMockData(ioc, goodPanda, `A table`, 'text/html'); + // addMockData(ioc, matPlotLib, matPlotLibResults, 'text/html'); + // const cursors = ['|', '/', '-', '\\']; + // let cursorPos = 0; + // let loops = 3; + // addContinuousMockData(ioc, spinningCursor, async _c => { + // const result = `${cursors[cursorPos]}\r`; + // cursorPos += 1; + // if (cursorPos >= cursors.length) { + // cursorPos = 0; + // loops -= 1; + // } + // return Promise.resolve({ result: result, haveMore: loops > 0 }); + // }); await addCode(ioc, wrapper, badPanda, true); verifyHtmlOnCell(wrapper, 'InteractiveCell', `has no attribute 'read'`, CellPosition.Last); - await addCode(ioc, wrapper, goodPanda); - verifyHtmlOnCell(wrapper, 'InteractiveCell', ``, CellPosition.Last); + // await addCode(ioc, wrapper, goodPanda); + // verifyHtmlOnCell(wrapper, 'InteractiveCell', ``, CellPosition.Last); - await addCode(ioc, wrapper, matPlotLib); - verifyHtmlOnCell(wrapper, 'InteractiveCell', /img|Figure/, CellPosition.Last); + // await addCode(ioc, wrapper, matPlotLib); + // verifyHtmlOnCell(wrapper, 'InteractiveCell', /img|Figure/, CellPosition.Last); - await addCode(ioc, wrapper, spinningCursor); - verifyHtmlOnCell(wrapper, 'InteractiveCell', '
', CellPosition.Last); + // await addCode(ioc, wrapper, spinningCursor); + // verifyHtmlOnCell(wrapper, 'InteractiveCell', '
', CellPosition.Last); addContinuousMockData(ioc, 'len?', async _c => { return Promise.resolve({ @@ -919,7 +919,7 @@ Type: builtin_function_or_method`, // Ignore white space. assert.equal( docManager.activeTextEditor.document.getText().trim(), - `# This file contains only the code required to produce the results of the gathered cell.\n${defaultCellMarker}\na=1\na` + `${defaultCellMarker} [markdown]\n## Gather not available` ); } }, @@ -952,7 +952,7 @@ Type: builtin_function_or_method`, // Ignore whitespace assert.equal( docManager.activeTextEditor.document.getText().trim(), - `# This file contains only the code required to produce the results of the gathered cell.\n${defaultCellMarker}\na=1\na` + `${defaultCellMarker} [markdown]\n## Gather not available` ); } }, diff --git a/src/test/datascience/interactiveWindowTestHelpers.tsx b/src/test/datascience/interactiveWindowTestHelpers.tsx index 28d96909779a..0fced0f03bb3 100644 --- a/src/test/datascience/interactiveWindowTestHelpers.tsx +++ b/src/test/datascience/interactiveWindowTestHelpers.tsx @@ -23,7 +23,15 @@ export function getInteractiveCellResults( export async function getOrCreateInteractiveWindow(ioc: DataScienceIocContainer): Promise { const interactiveWindowProvider = ioc.get(IInteractiveWindowProvider); - return (await interactiveWindowProvider.getOrCreateActive()) as InteractiveWindow; + const iw = (await interactiveWindowProvider.getOrCreateActive()) as InteractiveWindow; + + // const interactiveWindowListener = ioc.get(ICodeLensFactory); + // interactiveWindowListener.onMessage( + // InteractiveWindowMessages.NotebookExecutionActivated, + // Identifiers.InteractiveWindowIdentity + // ); + + return iw; } export function closeInteractiveWindow( diff --git a/src/test/datascience/nativeEditor.functional.test.tsx b/src/test/datascience/nativeEditor.functional.test.tsx index 621673befa3a..fbb31447864e 100644 --- a/src/test/datascience/nativeEditor.functional.test.tsx +++ b/src/test/datascience/nativeEditor.functional.test.tsx @@ -1230,12 +1230,12 @@ df.head()`; let imageButtons = cell.find(ImageButton); assert.equal(imageButtons.length, 6, 'Cell buttons not found'); const deleteButton = imageButtons.at(5); - await getNativeCellResults(ioc, wrapper, async () => { + const afterDelete = await getNativeCellResults(ioc, wrapper, async () => { deleteButton.simulate('click'); return Promise.resolve(); }); // Should have 3 cells - assert.equal(wrapper.find('NativeCell').length, 3, 'Cell not deleted'); + assert.equal(afterDelete.length, 3, 'Cell not deleted'); // Undo the delete await undo(); @@ -1254,12 +1254,12 @@ df.head()`; imageButtons = cell.find(ImageButton); assert.equal(imageButtons.length, 6, 'Cell buttons not found'); const moveUpButton = imageButtons.at(0); - await getNativeCellResults(ioc, wrapper, async () => { + const afterMove = await getNativeCellResults(ioc, wrapper, async () => { moveUpButton.simulate('click'); return Promise.resolve(); }); - let foundCell = getOutputCell(wrapper, 'NativeCell', 2)?.instance() as NativeCell; + let foundCell = getOutputCell(afterMove, 'NativeCell', 2)?.instance() as NativeCell; assert.equal(foundCell.props.cellVM.cell.id, 'NotebookImport#1', 'Cell did not move'); await undo(); foundCell = getOutputCell(wrapper, 'NativeCell', 2)?.instance() as NativeCell; From eab106bfee778f460c8a9163219e6c69ea463e58 Mon Sep 17 00:00:00 2001 From: Jim Griesmer Date: Thu, 5 Mar 2020 17:01:03 -0800 Subject: [PATCH 38/51] More fixes --- .vscode/launch.json | 4 ++-- src/test/datascience/dataScienceIocContainer.ts | 3 +++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index aa0d3d008b10..39f2c4888690 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -185,7 +185,7 @@ "--ui=tdd", "--recursive", "--colors", - //"--grep", "gathering", + //"--grep", "", "--timeout=300000" ], "outFiles": [ @@ -234,7 +234,7 @@ "--ui=tdd", "--recursive", "--colors", - "--grep", "Remove", + //"--grep", "", "--timeout=300000" ], "env": { diff --git a/src/test/datascience/dataScienceIocContainer.ts b/src/test/datascience/dataScienceIocContainer.ts index ce33940e22c4..684e4f172f7c 100644 --- a/src/test/datascience/dataScienceIocContainer.ts +++ b/src/test/datascience/dataScienceIocContainer.ts @@ -174,6 +174,7 @@ import { CodeWatcher } from '../../client/datascience/editor-integration/codewat import { DataScienceErrorHandler } from '../../client/datascience/errorHandler/errorHandler'; import { GatherProvider } from '../../client/datascience/gather/gather'; import { GatherListener } from '../../client/datascience/gather/gatherListener'; +import { GatherLogger } from '../../client/datascience/gather/gatherLogger'; import { IntellisenseProvider } from '../../client/datascience/interactive-common/intellisense/intellisenseProvider'; import { AutoSaveService } from '../../client/datascience/interactive-ipynb/autoSaveService'; import { NativeEditor } from '../../client/datascience/interactive-ipynb/nativeEditor'; @@ -226,6 +227,7 @@ import { IDataViewer, IDataViewerProvider, IDebugLocationTracker, + IGatherLogger, IGatherProvider, IInteractiveWindow, IInteractiveWindowListener, @@ -625,6 +627,7 @@ export class DataScienceIocContainer extends UnitTestIocContainer { this.serviceManager.addBinding(ICellHashLogger, INotebookExecutionLogger); this.serviceManager.addBinding(IJupyterDebugger, ICellHashListener); this.serviceManager.add(IGatherProvider, GatherProvider); + this.serviceManager.add(IGatherLogger, GatherLogger); this.serviceManager.addSingleton(ICodeLensFactory, CodeLensFactory); this.serviceManager.addSingleton(IShellDetector, TerminalNameShellDetector); this.serviceManager.addSingleton( From e7da45a05cebebc730eef0762c602d923009b03a Mon Sep 17 00:00:00 2001 From: Jim Griesmer Date: Thu, 5 Mar 2020 17:15:14 -0800 Subject: [PATCH 39/51] Whoops, fix IW functional test back to original --- .../interactiveWindow.functional.test.tsx | 40 +++++++++---------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/src/test/datascience/interactiveWindow.functional.test.tsx b/src/test/datascience/interactiveWindow.functional.test.tsx index 5cda6d02bc7c..611978139272 100644 --- a/src/test/datascience/interactiveWindow.functional.test.tsx +++ b/src/test/datascience/interactiveWindow.functional.test.tsx @@ -336,32 +336,32 @@ for _ in range(50): sys.stdout.write('\\r')`; addMockData(ioc, badPanda, `pandas has no attribute 'read'`, 'text/html', 'error'); - // addMockData(ioc, goodPanda, `A table`, 'text/html'); - // addMockData(ioc, matPlotLib, matPlotLibResults, 'text/html'); - // const cursors = ['|', '/', '-', '\\']; - // let cursorPos = 0; - // let loops = 3; - // addContinuousMockData(ioc, spinningCursor, async _c => { - // const result = `${cursors[cursorPos]}\r`; - // cursorPos += 1; - // if (cursorPos >= cursors.length) { - // cursorPos = 0; - // loops -= 1; - // } - // return Promise.resolve({ result: result, haveMore: loops > 0 }); - // }); + addMockData(ioc, goodPanda, `A table`, 'text/html'); + addMockData(ioc, matPlotLib, matPlotLibResults, 'text/html'); + const cursors = ['|', '/', '-', '\\']; + let cursorPos = 0; + let loops = 3; + addContinuousMockData(ioc, spinningCursor, async _c => { + const result = `${cursors[cursorPos]}\r`; + cursorPos += 1; + if (cursorPos >= cursors.length) { + cursorPos = 0; + loops -= 1; + } + return Promise.resolve({ result: result, haveMore: loops > 0 }); + }); await addCode(ioc, wrapper, badPanda, true); verifyHtmlOnCell(wrapper, 'InteractiveCell', `has no attribute 'read'`, CellPosition.Last); - // await addCode(ioc, wrapper, goodPanda); - // verifyHtmlOnCell(wrapper, 'InteractiveCell', ``, CellPosition.Last); + await addCode(ioc, wrapper, goodPanda); + verifyHtmlOnCell(wrapper, 'InteractiveCell', ``, CellPosition.Last); - // await addCode(ioc, wrapper, matPlotLib); - // verifyHtmlOnCell(wrapper, 'InteractiveCell', /img|Figure/, CellPosition.Last); + await addCode(ioc, wrapper, matPlotLib); + verifyHtmlOnCell(wrapper, 'InteractiveCell', /img|Figure/, CellPosition.Last); - // await addCode(ioc, wrapper, spinningCursor); - // verifyHtmlOnCell(wrapper, 'InteractiveCell', '
', CellPosition.Last); + await addCode(ioc, wrapper, spinningCursor); + verifyHtmlOnCell(wrapper, 'InteractiveCell', '
', CellPosition.Last); addContinuousMockData(ioc, 'len?', async _c => { return Promise.resolve({ From 4c43493b453fc859bc6beeade09ec18a0aacc9e9 Mon Sep 17 00:00:00 2001 From: Jim Griesmer Date: Thu, 5 Mar 2020 17:24:10 -0800 Subject: [PATCH 40/51] Temporarily include datascienceioccontainer in 120 character line length. --- .prettierrc.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.prettierrc.js b/.prettierrc.js index 49f6747c18bc..9c5f4642ba85 100644 --- a/.prettierrc.js +++ b/.prettierrc.js @@ -10,7 +10,7 @@ module.exports = { } }, { - files: ['**/datascience/serviceRegistry.ts', '**/datascience/dataScienceIocContainer.ts'], + files: ['**/datascience/serviceRegistry.ts'], options: { printWidth: 240 } From 681fc5646a9a18f6ccd5154d3d0f132ac26e61e0 Mon Sep 17 00:00:00 2001 From: Jim Griesmer Date: Thu, 5 Mar 2020 17:30:48 -0800 Subject: [PATCH 41/51] Make linter finally happy. --- src/test/datascience/interactiveWindowTestHelpers.tsx | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/test/datascience/interactiveWindowTestHelpers.tsx b/src/test/datascience/interactiveWindowTestHelpers.tsx index 0fced0f03bb3..3710f1ce36d9 100644 --- a/src/test/datascience/interactiveWindowTestHelpers.tsx +++ b/src/test/datascience/interactiveWindowTestHelpers.tsx @@ -23,15 +23,13 @@ export function getInteractiveCellResults( export async function getOrCreateInteractiveWindow(ioc: DataScienceIocContainer): Promise { const interactiveWindowProvider = ioc.get(IInteractiveWindowProvider); - const iw = (await interactiveWindowProvider.getOrCreateActive()) as InteractiveWindow; + return (await interactiveWindowProvider.getOrCreateActive()) as InteractiveWindow; // const interactiveWindowListener = ioc.get(ICodeLensFactory); // interactiveWindowListener.onMessage( // InteractiveWindowMessages.NotebookExecutionActivated, // Identifiers.InteractiveWindowIdentity // ); - - return iw; } export function closeInteractiveWindow( From 815c3cf9cba97ee2cd1ae7a5a313ef1c8e1aad7d Mon Sep 17 00:00:00 2001 From: Jim Griesmer Date: Thu, 5 Mar 2020 18:02:25 -0800 Subject: [PATCH 42/51] Fix Gather functional tests --- src/test/datascience/dataScienceIocContainer.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/test/datascience/dataScienceIocContainer.ts b/src/test/datascience/dataScienceIocContainer.ts index 684e4f172f7c..ee09259902ae 100644 --- a/src/test/datascience/dataScienceIocContainer.ts +++ b/src/test/datascience/dataScienceIocContainer.ts @@ -628,7 +628,9 @@ export class DataScienceIocContainer extends UnitTestIocContainer { this.serviceManager.addBinding(IJupyterDebugger, ICellHashListener); this.serviceManager.add(IGatherProvider, GatherProvider); this.serviceManager.add(IGatherLogger, GatherLogger); + this.serviceManager.addBinding(IGatherLogger, INotebookExecutionLogger); this.serviceManager.addSingleton(ICodeLensFactory, CodeLensFactory); + this.serviceManager.addBinding(ICodeLensFactory, IInteractiveWindowListener); this.serviceManager.addSingleton(IShellDetector, TerminalNameShellDetector); this.serviceManager.addSingleton( InterpeterHashProviderFactory, From 3d5135e829b5c613b539d85add83650375c5a071 Mon Sep 17 00:00:00 2001 From: Jim Griesmer Date: Fri, 6 Mar 2020 00:10:44 -0800 Subject: [PATCH 43/51] A bit more cleanup --- .npmrc | 2 -- build/webpack/webpack.extension.config.js | 5 +++-- .../editor-integration/cellhashprovider.ts | 18 +++++++++--------- .../datascience/gather/gatherListener.ts | 2 +- 4 files changed, 13 insertions(+), 14 deletions(-) diff --git a/.npmrc b/.npmrc index 6c091d85dc0a..16cc2ccdf1e8 100644 --- a/.npmrc +++ b/.npmrc @@ -1,3 +1 @@ @types:registry=https://registry.npmjs.org -@msrvida:registry=https://pkgs.dev.azure.com/msresearch/_packaging/MSR-Python-Analysis/npm/registry/ -always-auth=true diff --git a/build/webpack/webpack.extension.config.js b/build/webpack/webpack.extension.config.js index 8dfc9bc6c25f..64fabb1664ef 100644 --- a/build/webpack/webpack.extension.config.js +++ b/build/webpack/webpack.extension.config.js @@ -59,8 +59,9 @@ const config = { { enforce: 'post', test: /linebreak[\/\\]src[\/\\]linebreaker.js/, loader: 'transform-loader?brfs' } ] }, - // Externals essentially keeps webpack from trying to package them. The ppaPackageList variable is set to - // non-empty if the build pipeline has been authenticated to install @msrvida/python-program-analysis. + // Packages listed in externals keeps webpack from trying to package them. + // The ppaPackageList variable is set to non-empty if the build pipeline has been + //authenticated to install @msrvida/python-program-analysis. externals: ['vscode', 'commonjs', ...ppaPackageList, ...existingModulesInOutDir], plugins: [ ...common.getDefaultPlugins('extension'), diff --git a/src/client/datascience/editor-integration/cellhashprovider.ts b/src/client/datascience/editor-integration/cellhashprovider.ts index 28edf334fe82..9a53680a7831 100644 --- a/src/client/datascience/editor-integration/cellhashprovider.ts +++ b/src/client/datascience/editor-integration/cellhashprovider.ts @@ -7,7 +7,6 @@ import { Event, EventEmitter, Position, Range, TextDocumentChangeEvent, TextDocu import { splitMultilineString } from '../../../datascience-ui/common'; import { IDebugService, IDocumentManager } from '../../common/application/types'; -// tslint:disable-next-line: ordered-imports import { traceError, traceInfo } from '../../common/logger'; import { IFileSystem } from '../../common/platform/types'; import { IConfigurationService } from '../../common/types'; @@ -37,14 +36,6 @@ interface IRangedCellHash extends ICellHash { // hashes for cells. @injectable() export class CellHashProvider implements ICellHashProvider, IInteractiveWindowListener { - public get updated(): Event { - return this.updateEventEmitter.event; - } - - // tslint:disable-next-line: no-any - public get postMessage(): Event<{ message: string; payload: any }> { - return this.postEmitter.event; - } public executionCount: number = 0; // tslint:disable-next-line: no-any @@ -72,6 +63,15 @@ export class CellHashProvider implements ICellHashProvider, IInteractiveWindowLi this.hashes.clear(); } + public get updated(): Event { + return this.updateEventEmitter.event; + } + + // tslint:disable-next-line: no-any + public get postMessage(): Event<{ message: string; payload: any }> { + return this.postEmitter.event; + } + // tslint:disable-next-line: no-any public onMessage(message: string, payload?: any): void { switch (message) { diff --git a/src/client/datascience/gather/gatherListener.ts b/src/client/datascience/gather/gatherListener.ts index 77119b78cd27..312a15a8303f 100644 --- a/src/client/datascience/gather/gatherListener.ts +++ b/src/client/datascience/gather/gatherListener.ts @@ -37,9 +37,9 @@ export class GatherListener implements IInteractiveWindowListener { @inject(IApplicationShell) private applicationShell: IApplicationShell, @inject(INotebookExporter) private jupyterExporter: INotebookExporter, @inject(INotebookEditorProvider) private ipynbProvider: INotebookEditorProvider, - @inject(IConfigurationService) private configService: IConfigurationService, @inject(IJupyterExecution) private jupyterExecution: IJupyterExecution, @inject(IInteractiveWindowProvider) private interactiveWindowProvider: IInteractiveWindowProvider, + @inject(IConfigurationService) private configService: IConfigurationService, @inject(IDocumentManager) private documentManager: IDocumentManager, @inject(IFileSystem) private fileSystem: IFileSystem ) {} From 9c1b91b888d6d2c1a4cd784ac6f6dc5721f374c2 Mon Sep 17 00:00:00 2001 From: Jim Griesmer Date: Fri, 6 Mar 2020 13:53:42 -0800 Subject: [PATCH 44/51] React to PR feedback --- .../editor-integration/codeLensFactory.ts | 19 +++++++++++-- .../datascience/gather/gatherListener.ts | 18 ++++++++++++- src/client/datascience/gather/gatherLogger.ts | 2 +- .../datascience/jupyter/jupyterNotebook.ts | 27 +++---------------- .../datascience/jupyter/jupyterServer.ts | 8 +++--- .../jupyter/jupyterServerWrapper.ts | 10 +++---- .../jupyter/liveshare/guestJupyterNotebook.ts | 14 +++------- .../jupyter/liveshare/guestJupyterServer.ts | 4 ++- .../jupyter/liveshare/hostJupyterNotebook.ts | 6 ++--- .../jupyter/liveshare/hostJupyterServer.ts | 18 ++++--------- src/client/datascience/types.ts | 5 ++-- .../datascience/dataScienceIocContainer.ts | 18 +++++++------ .../gotocell.functional.test.ts | 1 - src/test/datascience/mockJupyterNotebook.ts | 4 +++ 14 files changed, 79 insertions(+), 75 deletions(-) diff --git a/src/client/datascience/editor-integration/codeLensFactory.ts b/src/client/datascience/editor-integration/codeLensFactory.ts index 4201f896f967..515ef7e9945a 100644 --- a/src/client/datascience/editor-integration/codeLensFactory.ts +++ b/src/client/datascience/editor-integration/codeLensFactory.ts @@ -14,13 +14,15 @@ import { CodeLensCommands, Commands } from '../constants'; import { InteractiveWindowMessages } from '../interactive-common/interactiveWindowTypes'; import { ICell, + ICellHashLogger, ICellHashProvider, ICodeLensFactory, IFileHashes, IInteractiveWindowListener, IInteractiveWindowProvider, IJupyterExecution, - INotebook + INotebook, + INotebookExecutionLogger } from '../types'; @injectable() @@ -125,13 +127,26 @@ export class CodeLensFactory implements ICodeLensFactory, IInteractiveWindowList // If we have an executing notebook, get its cell hash provider service. if (nb) { - this.hashProvider = nb.getCellHashProvider(); + this.hashProvider = this.getCellHashProvider(nb); if (this.hashProvider) { this.hashProvider.updated(this.hashesUpdated.bind(this)); } } } } + private getCellHashProvider(nb: INotebook): ICellHashProvider | undefined { + let cellHashLogger: ICellHashLogger | undefined; + + nb.getLoggers().forEach((logger: INotebookExecutionLogger) => { + if (logger as ICellHashLogger) { + cellHashLogger = logger; + } + }); + + if (cellHashLogger) { + return cellHashLogger.getProvider(); + } + } private enumerateCommands(resource: Resource): string[] { let fullCommandList: string[]; diff --git a/src/client/datascience/gather/gatherListener.ts b/src/client/datascience/gather/gatherListener.ts index 312a15a8303f..5ddfe794996e 100644 --- a/src/client/datascience/gather/gatherListener.ts +++ b/src/client/datascience/gather/gatherListener.ts @@ -13,14 +13,17 @@ import { Identifiers } from '../constants'; import { IInteractiveWindowMapping, InteractiveWindowMessages } from '../interactive-common/interactiveWindowTypes'; import { ICell, + IGatherLogger, IGatherProvider, IInteractiveWindowListener, IInteractiveWindowProvider, IJupyterExecution, INotebook, INotebookEditorProvider, + INotebookExecutionLogger, INotebookExporter } from '../types'; +import { GatherLogger } from './gatherLogger'; @injectable() export class GatherListener implements IInteractiveWindowListener { @@ -110,10 +113,23 @@ export class GatherListener implements IInteractiveWindowListener { // If we have an executing notebook, get its gather execution service. if (nb) { - this.gatherProvider = nb.getGatherProvider(); + this.gatherProvider = this.getGatherProvider(nb); } } } + private getGatherProvider(nb: INotebook): IGatherProvider | undefined { + let gatherLogger: IGatherLogger | undefined; + + nb.getLoggers().forEach((logger: INotebookExecutionLogger) => { + if (logger as IGatherLogger) { + gatherLogger = logger; + } + }); + + if (gatherLogger) { + return gatherLogger.getProvider(); + } + } private gatherCodeInternal = async (cell: ICell) => { const slicedProgram = this.gatherProvider ? this.gatherProvider.gatherCode(cell) : 'Gather internal error'; diff --git a/src/client/datascience/gather/gatherLogger.ts b/src/client/datascience/gather/gatherLogger.ts index 18ce7fc0cb0f..7759de59d033 100644 --- a/src/client/datascience/gather/gatherLogger.ts +++ b/src/client/datascience/gather/gatherLogger.ts @@ -36,7 +36,7 @@ export class GatherLogger implements IGatherLogger { } } - public service() { + public getProvider() { return this.gather; } } diff --git a/src/client/datascience/jupyter/jupyterNotebook.ts b/src/client/datascience/jupyter/jupyterNotebook.ts index 14bfa04293f2..723dc761bed5 100644 --- a/src/client/datascience/jupyter/jupyterNotebook.ts +++ b/src/client/datascience/jupyter/jupyterNotebook.ts @@ -20,7 +20,6 @@ import * as localize from '../../common/utils/localize'; import { noop } from '../../common/utils/misc'; import { StopWatch } from '../../common/utils/stopWatch'; import { PythonInterpreter } from '../../interpreter/contracts'; -import { IServiceContainer } from '../../ioc/types'; import { captureTelemetry, sendTelemetryEvent } from '../../telemetry'; import { generateCells } from '../cellFactory'; import { CellMatcher } from '../cellMatcher'; @@ -28,10 +27,6 @@ import { CodeSnippits, Identifiers, Telemetry } from '../constants'; import { CellState, ICell, - ICellHashLogger, - ICellHashProvider, - IGatherLogger, - IGatherProvider, IJupyterKernelSpec, IJupyterSession, INotebook, @@ -175,7 +170,7 @@ export class JupyterNotebookBase implements INotebook { private disposableRegistry: IDisposableRegistry, private owner: INotebookServer, private launchInfo: INotebookServerLaunchInfo, - serviceContainer: IServiceContainer, + loggers: INotebookExecutionLogger[], resource: Resource, identity: Uri, private getDisposedError: () => Error, @@ -193,7 +188,7 @@ export class JupyterNotebookBase implements INotebook { this.sessionStatusChanged = this.session.onSessionStatusChanged(statusChangeHandler); this._identity = identity; this._resource = resource; - this._loggers = serviceContainer.getAll(INotebookExecutionLogger); + this._loggers = [...loggers]; // Save our interpreter and don't change it. Later on when kernel changes // are possible, recompute it. } @@ -621,22 +616,8 @@ export class JupyterNotebookBase implements INotebook { this.kernelChanged.fire(spec); } - public getGatherProvider(): IGatherProvider | undefined { - const gatherLogger: INotebookExecutionLogger | undefined = this._loggers.find( - (logger: INotebookExecutionLogger) => { - return (logger).service !== undefined; - } - ); - return gatherLogger ? (gatherLogger).service() : undefined; - } - - public getCellHashProvider(): ICellHashProvider | undefined { - const cellHashLogger: INotebookExecutionLogger | undefined = this._loggers.find( - (logger: INotebookExecutionLogger) => { - return (logger).getProvider !== undefined; - } - ); - return cellHashLogger ? (cellHashLogger).getProvider() : undefined; + public getLoggers(): INotebookExecutionLogger[] { + return this._loggers; } private async initializeMatplotlib(cancelToken?: CancellationToken): Promise { diff --git a/src/client/datascience/jupyter/jupyterServer.ts b/src/client/datascience/jupyter/jupyterServer.ts index 3e92379d24ca..49f4fad38caa 100644 --- a/src/client/datascience/jupyter/jupyterServer.ts +++ b/src/client/datascience/jupyter/jupyterServer.ts @@ -18,13 +18,13 @@ import { import { createDeferred, Deferred } from '../../common/utils/async'; import * as localize from '../../common/utils/localize'; import { noop } from '../../common/utils/misc'; -import { IServiceContainer } from '../../ioc/types'; import { IConnection, IJupyterSession, IJupyterSessionManager, IJupyterSessionManagerFactory, INotebook, + INotebookExecutionLogger, INotebookServer, INotebookServerLaunchInfo } from '../types'; @@ -48,7 +48,7 @@ export class JupyterServerBase implements INotebookServer { private disposableRegistry: IDisposableRegistry, protected readonly configService: IConfigurationService, private sessionManagerFactory: IJupyterSessionManagerFactory, - private serviceContainer: IServiceContainer, + private loggers: INotebookExecutionLogger[], private jupyterOutputChannel: IOutputChannel ) { this.asyncRegistry.push(this); @@ -111,7 +111,7 @@ export class JupyterServerBase implements INotebookServer { savedSession, this.disposableRegistry, this.configService, - this.serviceContainer, + this.loggers, notebookMetadata, cancelToken ).then(r => { @@ -219,7 +219,7 @@ export class JupyterServerBase implements INotebookServer { _savedSession: IJupyterSession | undefined, _disposableRegistry: IDisposableRegistry, _configService: IConfigurationService, - _serviceContainer: IServiceContainer, + _loggers: INotebookExecutionLogger[], _notebookMetadata?: nbformat.INotebookMetadata, _cancelToken?: CancellationToken ): Promise { diff --git a/src/client/datascience/jupyter/jupyterServerWrapper.ts b/src/client/datascience/jupyter/jupyterServerWrapper.ts index b40a0d0c5222..b9c9a0643902 100644 --- a/src/client/datascience/jupyter/jupyterServerWrapper.ts +++ b/src/client/datascience/jupyter/jupyterServerWrapper.ts @@ -2,7 +2,7 @@ // Licensed under the MIT License. 'use strict'; import { nbformat } from '@jupyterlab/coreutils'; -import { inject, injectable, named } from 'inversify'; +import { inject, injectable, multiInject, named, optional } from 'inversify'; import * as uuid from 'uuid/v4'; import { Uri } from 'vscode'; import { CancellationToken } from 'vscode-jsonrpc'; @@ -18,13 +18,13 @@ import { Resource } from '../../common/types'; import { IInterpreterService } from '../../interpreter/contracts'; -import { IServiceContainer } from '../../ioc/types'; import { JUPYTER_OUTPUT_CHANNEL } from '../constants'; import { IConnection, IDataScience, IJupyterSessionManagerFactory, INotebook, + INotebookExecutionLogger, INotebookServer, INotebookServerLaunchInfo } from '../types'; @@ -46,7 +46,7 @@ type JupyterServerClassType = { configService: IConfigurationService, sessionManager: IJupyterSessionManagerFactory, workspaceService: IWorkspaceService, - serviceContainer: IServiceContainer, + loggers: INotebookExecutionLogger[], appShell: IApplicationShell, fs: IFileSystem, kernelSelector: KernelSelector, @@ -73,7 +73,7 @@ export class JupyterServerWrapper implements INotebookServer, ILiveShareHasRole @inject(IConfigurationService) configService: IConfigurationService, @inject(IJupyterSessionManagerFactory) sessionManager: IJupyterSessionManagerFactory, @inject(IWorkspaceService) workspaceService: IWorkspaceService, - @inject(IServiceContainer) serviceContainer: IServiceContainer, + @multiInject(INotebookExecutionLogger) @optional() loggers: INotebookExecutionLogger[] | undefined, @inject(IApplicationShell) appShell: IApplicationShell, @inject(IFileSystem) fs: IFileSystem, @inject(IInterpreterService) interpreterService: IInterpreterService, @@ -93,7 +93,7 @@ export class JupyterServerWrapper implements INotebookServer, ILiveShareHasRole configService, sessionManager, workspaceService, - serviceContainer, + loggers ? loggers : [], appShell, fs, kernelSelector, diff --git a/src/client/datascience/jupyter/liveshare/guestJupyterNotebook.ts b/src/client/datascience/jupyter/liveshare/guestJupyterNotebook.ts index ad1b77976e89..d7938449b247 100644 --- a/src/client/datascience/jupyter/liveshare/guestJupyterNotebook.ts +++ b/src/client/datascience/jupyter/liveshare/guestJupyterNotebook.ts @@ -18,11 +18,10 @@ import { PythonInterpreter } from '../../../interpreter/contracts'; import { LiveShare, LiveShareCommands } from '../../constants'; import { ICell, - ICellHashProvider, - IGatherProvider, IJupyterKernelSpec, INotebook, INotebookCompletion, + INotebookExecutionLogger, INotebookServer, InterruptResult } from '../../types'; @@ -235,17 +234,12 @@ export class GuestJupyterNotebook return; } - public getGatherProvider(): IGatherProvider | undefined { - return; - } - - public getCellHashProvider(): ICellHashProvider | undefined { - return; - } - public setKernelSpec(_spec: IJupyterKernelSpec | LiveKernelModel, _timeout: number): Promise { return Promise.resolve(); } + public getLoggers(): INotebookExecutionLogger[] { + return []; + } private onServerResponse = (args: Object) => { const er = args as IExecuteObservableResponse; diff --git a/src/client/datascience/jupyter/liveshare/guestJupyterServer.ts b/src/client/datascience/jupyter/liveshare/guestJupyterServer.ts index b1783f4c4054..79ec86f16be3 100644 --- a/src/client/datascience/jupyter/liveshare/guestJupyterServer.ts +++ b/src/client/datascience/jupyter/liveshare/guestJupyterServer.ts @@ -15,6 +15,7 @@ import { IDataScience, IJupyterSessionManagerFactory, INotebook, + INotebookExecutionLogger, INotebookServer, INotebookServerLaunchInfo } from '../../types'; @@ -37,7 +38,8 @@ export class GuestJupyterServer private disposableRegistry: IDisposableRegistry, private configService: IConfigurationService, _sessionManager: IJupyterSessionManagerFactory, - _workspaceService: IWorkspaceService + _workspaceService: IWorkspaceService, + _loggers: INotebookExecutionLogger[] ) { super(liveShare); } diff --git a/src/client/datascience/jupyter/liveshare/hostJupyterNotebook.ts b/src/client/datascience/jupyter/liveshare/hostJupyterNotebook.ts index 4e6fa86d10c2..9eef63c72f46 100644 --- a/src/client/datascience/jupyter/liveshare/hostJupyterNotebook.ts +++ b/src/client/datascience/jupyter/liveshare/hostJupyterNotebook.ts @@ -11,13 +11,13 @@ import { traceError } from '../../../common/logger'; import { IFileSystem } from '../../../common/platform/types'; import { IConfigurationService, IDisposableRegistry, Resource } from '../../../common/types'; import { createDeferred } from '../../../common/utils/async'; -import { IServiceContainer } from '../../../ioc/types'; import { Identifiers, LiveShare, LiveShareCommands } from '../../constants'; import { IExecuteInfo } from '../../interactive-common/interactiveWindowTypes'; import { ICell, IJupyterSession, INotebook, + INotebookExecutionLogger, INotebookServer, INotebookServerLaunchInfo, InterruptResult @@ -48,7 +48,7 @@ export class HostJupyterNotebook disposableRegistry: IDisposableRegistry, owner: INotebookServer, launchInfo: INotebookServerLaunchInfo, - serviceContainer: IServiceContainer, + loggers: INotebookExecutionLogger[], resource: Resource, identity: vscode.Uri, getDisposedError: () => Error, @@ -63,7 +63,7 @@ export class HostJupyterNotebook disposableRegistry, owner, launchInfo, - serviceContainer, + loggers, resource, identity, getDisposedError, diff --git a/src/client/datascience/jupyter/liveshare/hostJupyterServer.ts b/src/client/datascience/jupyter/liveshare/hostJupyterServer.ts index 2b21b31dc1fc..b510f2be3e13 100644 --- a/src/client/datascience/jupyter/liveshare/hostJupyterServer.ts +++ b/src/client/datascience/jupyter/liveshare/hostJupyterServer.ts @@ -21,7 +21,6 @@ import { } from '../../../common/types'; import * as localize from '../../../common/utils/localize'; import { IInterpreterService } from '../../../interpreter/contracts'; -import { IServiceContainer } from '../../../ioc/types'; import { Identifiers, LiveShare, LiveShareCommands, RegExpValues } from '../../constants'; import { IDataScience, @@ -29,6 +28,7 @@ import { IJupyterSessionManager, IJupyterSessionManagerFactory, INotebook, + INotebookExecutionLogger, INotebookServer, INotebookServerLaunchInfo } from '../../types'; @@ -54,22 +54,14 @@ export class HostJupyterServer extends LiveShareParticipantHost(JupyterServerBas configService: IConfigurationService, sessionManager: IJupyterSessionManagerFactory, private workspaceService: IWorkspaceService, - serviceContainer: IServiceContainer, + loggers: INotebookExecutionLogger[], private appService: IApplicationShell, private fs: IFileSystem, private readonly kernelSelector: KernelSelector, private readonly interpreterService: IInterpreterService, outputChannel: IOutputChannel ) { - super( - liveShare, - asyncRegistry, - disposableRegistry, - configService, - sessionManager, - serviceContainer, - outputChannel - ); + super(liveShare, asyncRegistry, disposableRegistry, configService, sessionManager, loggers, outputChannel); } public async dispose(): Promise { @@ -170,7 +162,7 @@ export class HostJupyterServer extends LiveShareParticipantHost(JupyterServerBas possibleSession: IJupyterSession | undefined, disposableRegistry: IDisposableRegistry, configService: IConfigurationService, - serviceContainer: IServiceContainer, + loggers: INotebookExecutionLogger[], notebookMetadata?: nbformat.INotebookMetadata, cancelToken?: CancellationToken ): Promise { @@ -215,7 +207,7 @@ export class HostJupyterServer extends LiveShareParticipantHost(JupyterServerBas disposableRegistry, this, info, - serviceContainer, + loggers, resource, identity, this.getDisposedError.bind(this), diff --git a/src/client/datascience/types.ts b/src/client/datascience/types.ts index ac46efd1a82a..395c721188de 100644 --- a/src/client/datascience/types.ts +++ b/src/client/datascience/types.ts @@ -140,8 +140,7 @@ export interface INotebook extends IAsyncDisposable { getKernelSpec(): IJupyterKernelSpec | LiveKernelModel | undefined; setKernelSpec(spec: IJupyterKernelSpec | LiveKernelModel, timeoutMS: number): Promise; setInterpreter(interpeter: PythonInterpreter): void; - getGatherProvider(): IGatherProvider | undefined; - getCellHashProvider(): ICellHashProvider | undefined; + getLoggers(): INotebookExecutionLogger[]; } export interface INotebookServerOptions { @@ -171,7 +170,7 @@ export interface IGatherProvider { export const IGatherLogger = Symbol('IGatherLogger'); export interface IGatherLogger extends INotebookExecutionLogger { - service(): IGatherProvider; + getProvider(): IGatherProvider; } export const IJupyterExecution = Symbol('IJupyterExecution'); diff --git a/src/test/datascience/dataScienceIocContainer.ts b/src/test/datascience/dataScienceIocContainer.ts index ee09259902ae..8ec338021b60 100644 --- a/src/test/datascience/dataScienceIocContainer.ts +++ b/src/test/datascience/dataScienceIocContainer.ts @@ -517,7 +517,9 @@ export class DataScienceIocContainer extends UnitTestIocContainer { this.serviceManager.add(IDataScienceErrorHandler, DataScienceErrorHandler); this.serviceManager.add(IInstallationChannelManager, InstallationChannelManager); this.serviceManager.addSingleton(IJupyterVariables, JupyterVariables); - this.serviceManager.addSingleton(IJupyterDebugger, JupyterDebugger); + this.serviceManager.addSingleton(IJupyterDebugger, JupyterDebugger, undefined, [ + ICellHashListener + ]); this.serviceManager.addSingleton(IDebugLocationTracker, DebugLocationTrackerFactory); this.serviceManager.addSingleton(INotebookEditorProvider, TestNativeEditorProvider); this.serviceManager.addSingleton( @@ -623,14 +625,14 @@ export class DataScienceIocContainer extends UnitTestIocContainer { this.serviceManager.add(IProtocolParser, ProtocolParser); this.serviceManager.addSingleton(IDebugService, MockDebuggerService); this.serviceManager.add(ICellHashProvider, CellHashProvider); - this.serviceManager.add(ICellHashLogger, CellHashLogger); - this.serviceManager.addBinding(ICellHashLogger, INotebookExecutionLogger); - this.serviceManager.addBinding(IJupyterDebugger, ICellHashListener); + this.serviceManager.add(ICellHashLogger, CellHashLogger, undefined, [ + INotebookExecutionLogger + ]); this.serviceManager.add(IGatherProvider, GatherProvider); - this.serviceManager.add(IGatherLogger, GatherLogger); - this.serviceManager.addBinding(IGatherLogger, INotebookExecutionLogger); - this.serviceManager.addSingleton(ICodeLensFactory, CodeLensFactory); - this.serviceManager.addBinding(ICodeLensFactory, IInteractiveWindowListener); + this.serviceManager.add(IGatherLogger, GatherLogger, undefined, [INotebookExecutionLogger]); + this.serviceManager.addSingleton(ICodeLensFactory, CodeLensFactory, undefined, [ + IInteractiveWindowListener + ]); this.serviceManager.addSingleton(IShellDetector, TerminalNameShellDetector); this.serviceManager.addSingleton( InterpeterHashProviderFactory, diff --git a/src/test/datascience/editor-integration/gotocell.functional.test.ts b/src/test/datascience/editor-integration/gotocell.functional.test.ts index ad54c2e35f00..017445514491 100644 --- a/src/test/datascience/editor-integration/gotocell.functional.test.ts +++ b/src/test/datascience/editor-integration/gotocell.functional.test.ts @@ -103,7 +103,6 @@ suite('DataScience gotocell tests', () => { undefined, Uri.parse(Identifiers.InteractiveWindowIdentity) ); - nb.getCellHashProvider(); const listener = (codeLensFactory as any) as IInteractiveWindowListener; listener.onMessage( InteractiveWindowMessages.NotebookExecutionActivated, diff --git a/src/test/datascience/mockJupyterNotebook.ts b/src/test/datascience/mockJupyterNotebook.ts index 86f3b2d03ca0..0743395b83cd 100644 --- a/src/test/datascience/mockJupyterNotebook.ts +++ b/src/test/datascience/mockJupyterNotebook.ts @@ -132,4 +132,8 @@ export class MockJupyterNotebook implements INotebook { public setKernelSpec(_spec: IJupyterKernelSpec | LiveKernelModel, _timeout: number): Promise { return Promise.resolve(); } + + public getLoggers(): INotebookExecutionLogger[] { + return []; + } } From 89517a478ef5a46d91167f75e6fcce87917c2eb3 Mon Sep 17 00:00:00 2001 From: Jim Griesmer Date: Sat, 7 Mar 2020 00:01:55 -0800 Subject: [PATCH 45/51] Couple more PR review changes. --- .../datascience/editor-integration/cellhashprovider.ts | 2 +- src/test/datascience/interactiveWindowTestHelpers.tsx | 6 ------ 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/src/client/datascience/editor-integration/cellhashprovider.ts b/src/client/datascience/editor-integration/cellhashprovider.ts index 9a53680a7831..25d738c7580e 100644 --- a/src/client/datascience/editor-integration/cellhashprovider.ts +++ b/src/client/datascience/editor-integration/cellhashprovider.ts @@ -36,7 +36,7 @@ interface IRangedCellHash extends ICellHash { // hashes for cells. @injectable() export class CellHashProvider implements ICellHashProvider, IInteractiveWindowListener { - public executionCount: number = 0; + public readonly executionCount: number = 0; // tslint:disable-next-line: no-any private postEmitter: EventEmitter<{ message: string; payload: any }> = new EventEmitter<{ diff --git a/src/test/datascience/interactiveWindowTestHelpers.tsx b/src/test/datascience/interactiveWindowTestHelpers.tsx index 3710f1ce36d9..28d96909779a 100644 --- a/src/test/datascience/interactiveWindowTestHelpers.tsx +++ b/src/test/datascience/interactiveWindowTestHelpers.tsx @@ -24,12 +24,6 @@ export function getInteractiveCellResults( export async function getOrCreateInteractiveWindow(ioc: DataScienceIocContainer): Promise { const interactiveWindowProvider = ioc.get(IInteractiveWindowProvider); return (await interactiveWindowProvider.getOrCreateActive()) as InteractiveWindow; - - // const interactiveWindowListener = ioc.get(ICodeLensFactory); - // interactiveWindowListener.onMessage( - // InteractiveWindowMessages.NotebookExecutionActivated, - // Identifiers.InteractiveWindowIdentity - // ); } export function closeInteractiveWindow( From 5467f3fbead473ea01c058be7d1f1b42c73b695f Mon Sep 17 00:00:00 2001 From: Jim Griesmer Date: Sat, 7 Mar 2020 00:12:20 -0800 Subject: [PATCH 46/51] Tweak --- src/client/datascience/serviceRegistry.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/client/datascience/serviceRegistry.ts b/src/client/datascience/serviceRegistry.ts index 70e4341bb557..d541d69319a8 100644 --- a/src/client/datascience/serviceRegistry.ts +++ b/src/client/datascience/serviceRegistry.ts @@ -120,6 +120,7 @@ import { // tslint:disable-next-line: max-func-body-length export function registerTypes(serviceManager: IServiceManager) { const useCustomEditorApi = serviceManager.get(IApplicationEnvironment).packageJson.enableProposedApi; + serviceManager.addSingletonInstance(UseCustomEditorApi, useCustomEditorApi); serviceManager.add(ICellHashLogger, CellHashLogger, undefined, [INotebookExecutionLogger]); serviceManager.add(ICellHashProvider, CellHashProvider); @@ -188,7 +189,6 @@ export function registerTypes(serviceManager: IServiceManager) { serviceManager.addSingleton(KernelSwitcherCommand, KernelSwitcherCommand); serviceManager.addSingleton(NotebookStarter, NotebookStarter); serviceManager.addSingleton(ProgressReporter, ProgressReporter); - serviceManager.addSingletonInstance(UseCustomEditorApi, useCustomEditorApi); // Temporary code, to allow users to revert to the old behavior. const cfg = serviceManager.get(IWorkspaceService).getConfiguration('python.dataScience', undefined); From 535bb53810362636cbcf52efab7fd74a7fb34777 Mon Sep 17 00:00:00 2001 From: Jim Griesmer Date: Sat, 7 Mar 2020 02:26:18 -0800 Subject: [PATCH 47/51] More minor cleanup --- .../datascience/editor-integration/cellhashLogger.ts | 4 ++-- .../editor-integration/cellhashprovider.ts | 11 +++++++++-- src/client/datascience/gather/gatherListener.ts | 1 - src/client/datascience/types.ts | 2 ++ 4 files changed, 13 insertions(+), 5 deletions(-) diff --git a/src/client/datascience/editor-integration/cellhashLogger.ts b/src/client/datascience/editor-integration/cellhashLogger.ts index 3d3fd799fd8f..aa03183e6762 100644 --- a/src/client/datascience/editor-integration/cellhashLogger.ts +++ b/src/client/datascience/editor-integration/cellhashLogger.ts @@ -24,11 +24,11 @@ export class CellHashLogger implements ICellHashLogger { const stripped = providerObj.extractExecutableLines(cell); if (stripped.length > 0 && stripped.find(s => s.trim().length > 0)) { // When the user adds new code, we know the execution count is increasing - providerObj.executionCount += 1; + providerObj.incExecutionCount(); // Skip hash on unknown file though if (cell.file !== Identifiers.EmptyFileName) { - await providerObj.addCellHash(cell, providerObj.executionCount); + await providerObj.addCellHash(cell, providerObj.getExecutionCount()); } } } diff --git a/src/client/datascience/editor-integration/cellhashprovider.ts b/src/client/datascience/editor-integration/cellhashprovider.ts index 25d738c7580e..934507c2eb3e 100644 --- a/src/client/datascience/editor-integration/cellhashprovider.ts +++ b/src/client/datascience/editor-integration/cellhashprovider.ts @@ -36,8 +36,6 @@ interface IRangedCellHash extends ICellHash { // hashes for cells. @injectable() export class CellHashProvider implements ICellHashProvider, IInteractiveWindowListener { - public readonly executionCount: number = 0; - // tslint:disable-next-line: no-any private postEmitter: EventEmitter<{ message: string; payload: any }> = new EventEmitter<{ message: string; @@ -45,6 +43,7 @@ export class CellHashProvider implements ICellHashProvider, IInteractiveWindowLi payload: any; }>(); // Map of file to Map of start line to actual hash + private executionCount: number = 0; private hashes: Map = new Map(); private updateEventEmitter: EventEmitter = new EventEmitter(); @@ -235,6 +234,14 @@ export class CellHashProvider implements ICellHashProvider, IInteractiveWindowLi } } + public getExecutionCount(): number { + return this.executionCount; + } + + public incExecutionCount(): void { + this.executionCount += 1; + } + private onChangedDocument(e: TextDocumentChangeEvent) { // See if the document is in our list of docs to watch const perFile = this.hashes.get(e.document.fileName); diff --git a/src/client/datascience/gather/gatherListener.ts b/src/client/datascience/gather/gatherListener.ts index 5ddfe794996e..5a0b1149c900 100644 --- a/src/client/datascience/gather/gatherListener.ts +++ b/src/client/datascience/gather/gatherListener.ts @@ -23,7 +23,6 @@ import { INotebookExecutionLogger, INotebookExporter } from '../types'; -import { GatherLogger } from './gatherLogger'; @injectable() export class GatherListener implements IInteractiveWindowListener { diff --git a/src/client/datascience/types.ts b/src/client/datascience/types.ts index 395c721188de..81ff5dc1b3ed 100644 --- a/src/client/datascience/types.ts +++ b/src/client/datascience/types.ts @@ -696,6 +696,8 @@ export const ICellHashProvider = Symbol('ICellHashProvider'); export interface ICellHashProvider { updated: Event; getHashes(): IFileHashes[]; + getExecutionCount(): number; + incExecutionCount(): void; } export const ICellHashLogger = Symbol('ICellHashLogger'); From cf41937c17b05f566b2a555fd671f1f5e09c486d Mon Sep 17 00:00:00 2001 From: Jim Griesmer Date: Sat, 7 Mar 2020 15:01:20 -0800 Subject: [PATCH 48/51] Fix provider finders. --- .../datascience/editor-integration/codeLensFactory.ts | 10 +++------- src/client/datascience/gather/gatherListener.ts | 10 +++------- 2 files changed, 6 insertions(+), 14 deletions(-) diff --git a/src/client/datascience/editor-integration/codeLensFactory.ts b/src/client/datascience/editor-integration/codeLensFactory.ts index 515ef7e9945a..4b9a9def0a75 100644 --- a/src/client/datascience/editor-integration/codeLensFactory.ts +++ b/src/client/datascience/editor-integration/codeLensFactory.ts @@ -135,13 +135,9 @@ export class CodeLensFactory implements ICodeLensFactory, IInteractiveWindowList } } private getCellHashProvider(nb: INotebook): ICellHashProvider | undefined { - let cellHashLogger: ICellHashLogger | undefined; - - nb.getLoggers().forEach((logger: INotebookExecutionLogger) => { - if (logger as ICellHashLogger) { - cellHashLogger = logger; - } - }); + const cellHashLogger = ( + nb.getLoggers().find((logger: INotebookExecutionLogger) => logger) + ); if (cellHashLogger) { return cellHashLogger.getProvider(); diff --git a/src/client/datascience/gather/gatherListener.ts b/src/client/datascience/gather/gatherListener.ts index 5a0b1149c900..e9deb8632ce1 100644 --- a/src/client/datascience/gather/gatherListener.ts +++ b/src/client/datascience/gather/gatherListener.ts @@ -117,13 +117,9 @@ export class GatherListener implements IInteractiveWindowListener { } } private getGatherProvider(nb: INotebook): IGatherProvider | undefined { - let gatherLogger: IGatherLogger | undefined; - - nb.getLoggers().forEach((logger: INotebookExecutionLogger) => { - if (logger as IGatherLogger) { - gatherLogger = logger; - } - }); + const gatherLogger = ( + nb.getLoggers().find((logger: INotebookExecutionLogger) => logger) + ); if (gatherLogger) { return gatherLogger.getProvider(); From 63e017536749267bc839de07fe2960fdf4a93019 Mon Sep 17 00:00:00 2001 From: Jim Griesmer Date: Sat, 7 Mar 2020 16:08:22 -0800 Subject: [PATCH 49/51] Actually fix the provider get problem. --- src/client/datascience/editor-integration/cellhashLogger.ts | 2 +- src/client/datascience/editor-integration/codeLensFactory.ts | 4 ++-- src/client/datascience/gather/gatherListener.ts | 5 +++-- src/client/datascience/gather/gatherLogger.ts | 2 +- src/client/datascience/types.ts | 4 ++-- 5 files changed, 9 insertions(+), 8 deletions(-) diff --git a/src/client/datascience/editor-integration/cellhashLogger.ts b/src/client/datascience/editor-integration/cellhashLogger.ts index aa03183e6762..262e8e0bb16f 100644 --- a/src/client/datascience/editor-integration/cellhashLogger.ts +++ b/src/client/datascience/editor-integration/cellhashLogger.ts @@ -42,7 +42,7 @@ export class CellHashLogger implements ICellHashLogger { noop(); } - public getProvider(): ICellHashProvider { + public getCellHashProvider(): ICellHashProvider { return this.provider; } } diff --git a/src/client/datascience/editor-integration/codeLensFactory.ts b/src/client/datascience/editor-integration/codeLensFactory.ts index 4b9a9def0a75..459d41ccc79d 100644 --- a/src/client/datascience/editor-integration/codeLensFactory.ts +++ b/src/client/datascience/editor-integration/codeLensFactory.ts @@ -136,11 +136,11 @@ export class CodeLensFactory implements ICodeLensFactory, IInteractiveWindowList } private getCellHashProvider(nb: INotebook): ICellHashProvider | undefined { const cellHashLogger = ( - nb.getLoggers().find((logger: INotebookExecutionLogger) => logger) + nb.getLoggers().find((logger: INotebookExecutionLogger) => (logger).getCellHashProvider) ); if (cellHashLogger) { - return cellHashLogger.getProvider(); + return cellHashLogger.getCellHashProvider(); } } diff --git a/src/client/datascience/gather/gatherListener.ts b/src/client/datascience/gather/gatherListener.ts index e9deb8632ce1..1d2a1b4f57ae 100644 --- a/src/client/datascience/gather/gatherListener.ts +++ b/src/client/datascience/gather/gatherListener.ts @@ -116,13 +116,14 @@ export class GatherListener implements IInteractiveWindowListener { } } } + private getGatherProvider(nb: INotebook): IGatherProvider | undefined { const gatherLogger = ( - nb.getLoggers().find((logger: INotebookExecutionLogger) => logger) + nb.getLoggers().find((logger: INotebookExecutionLogger) => (logger).getGatherProvider) ); if (gatherLogger) { - return gatherLogger.getProvider(); + return gatherLogger.getGatherProvider(); } } diff --git a/src/client/datascience/gather/gatherLogger.ts b/src/client/datascience/gather/gatherLogger.ts index 7759de59d033..ffa85f006547 100644 --- a/src/client/datascience/gather/gatherLogger.ts +++ b/src/client/datascience/gather/gatherLogger.ts @@ -36,7 +36,7 @@ export class GatherLogger implements IGatherLogger { } } - public getProvider() { + public getGatherProvider() { return this.gather; } } diff --git a/src/client/datascience/types.ts b/src/client/datascience/types.ts index 81ff5dc1b3ed..644bab30377c 100644 --- a/src/client/datascience/types.ts +++ b/src/client/datascience/types.ts @@ -170,7 +170,7 @@ export interface IGatherProvider { export const IGatherLogger = Symbol('IGatherLogger'); export interface IGatherLogger extends INotebookExecutionLogger { - getProvider(): IGatherProvider; + getGatherProvider(): IGatherProvider; } export const IJupyterExecution = Symbol('IJupyterExecution'); @@ -702,7 +702,7 @@ export interface ICellHashProvider { export const ICellHashLogger = Symbol('ICellHashLogger'); export interface ICellHashLogger extends INotebookExecutionLogger { - getProvider(): ICellHashProvider; + getCellHashProvider(): ICellHashProvider; } export interface IDebugLocation { From 2f72957ae6da350a282a37a6952a4b71c91f2284 Mon Sep 17 00:00:00 2001 From: Jim Griesmer Date: Sun, 8 Mar 2020 00:38:59 -0800 Subject: [PATCH 50/51] Fix smoke test --- src/client/datascience/gather/gather.ts | 19 ++++++++----------- src/client/datascience/serviceRegistry.ts | 1 - 2 files changed, 8 insertions(+), 12 deletions(-) diff --git a/src/client/datascience/gather/gather.ts b/src/client/datascience/gather/gather.ts index 951fb88937b5..42fe255f0516 100644 --- a/src/client/datascience/gather/gather.ts +++ b/src/client/datascience/gather/gather.ts @@ -32,21 +32,18 @@ export class GatherProvider implements IGatherProvider { : false; if (this._enabled) { - try { - // tslint:disable-next-line: no-require-imports - const ppa = require('@msrvida/python-program-analysis') as typeof import('@msrvida-python-program-analysis'); + // tslint:disable-next-line: no-require-imports + const ppa = require('@msrvida/python-program-analysis') as typeof import('@msrvida-python-program-analysis'); + if (ppa) { this.dataflowAnalyzer = new ppa.DataflowAnalyzer(); this._executionSlicer = new ppa.ExecutionLogSlicer(this.dataflowAnalyzer); - if (this._enabled) { - this.disposables.push( - this.configService.getSettings(undefined).onDidChange(e => this.updateEnableGather(e)) - ); - } - traceInfo('Gathering tools have been activated'); - } catch (ex) { - traceInfo('Gathering tools could not be activated'); + this.disposables.push( + this.configService.getSettings(undefined).onDidChange(e => this.updateEnableGather(e)) + ); + } else { + traceInfo('Gathering tools could not be activated. Indicates build of VSIX was not'); } } } diff --git a/src/client/datascience/serviceRegistry.ts b/src/client/datascience/serviceRegistry.ts index d541d69319a8..0fcd8fb9f30c 100644 --- a/src/client/datascience/serviceRegistry.ts +++ b/src/client/datascience/serviceRegistry.ts @@ -145,7 +145,6 @@ export function registerTypes(serviceManager: IServiceManager) { serviceManager.addSingleton(CellOutputMimeTypeTracker, CellOutputMimeTypeTracker, undefined, [IExtensionSingleActivationService, INotebookExecutionLogger]); serviceManager.addSingleton(CommandRegistry, CommandRegistry); serviceManager.addSingleton(DataViewerDependencyService, DataViewerDependencyService); - serviceManager.addSingleton(DataViewerDependencyService, DataViewerDependencyService); serviceManager.addSingleton(ICodeCssGenerator, CodeCssGenerator); serviceManager.addSingleton(ICodeLensFactory, CodeLensFactory, undefined, [IInteractiveWindowListener]); serviceManager.addSingleton(IDataScience, DataScience); From 1de83b900a9aec02fdf347e96e76039e8179b983 Mon Sep 17 00:00:00 2001 From: Jim Griesmer Date: Sun, 8 Mar 2020 01:03:04 -0800 Subject: [PATCH 51/51] Fix unit test --- src/client/datascience/gather/gather.ts | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/src/client/datascience/gather/gather.ts b/src/client/datascience/gather/gather.ts index 42fe255f0516..77aca9dddcbb 100644 --- a/src/client/datascience/gather/gather.ts +++ b/src/client/datascience/gather/gather.ts @@ -32,17 +32,19 @@ export class GatherProvider implements IGatherProvider { : false; if (this._enabled) { - // tslint:disable-next-line: no-require-imports - const ppa = require('@msrvida/python-program-analysis') as typeof import('@msrvida-python-program-analysis'); - - if (ppa) { - this.dataflowAnalyzer = new ppa.DataflowAnalyzer(); - this._executionSlicer = new ppa.ExecutionLogSlicer(this.dataflowAnalyzer); - - this.disposables.push( - this.configService.getSettings(undefined).onDidChange(e => this.updateEnableGather(e)) - ); - } else { + try { + // tslint:disable-next-line: no-require-imports + const ppa = require('@msrvida/python-program-analysis') as typeof import('@msrvida-python-program-analysis'); + + if (ppa) { + this.dataflowAnalyzer = new ppa.DataflowAnalyzer(); + this._executionSlicer = new ppa.ExecutionLogSlicer(this.dataflowAnalyzer); + + this.disposables.push( + this.configService.getSettings(undefined).onDidChange(e => this.updateEnableGather(e)) + ); + } + } catch (ex) { traceInfo('Gathering tools could not be activated. Indicates build of VSIX was not'); } }