Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
78 commits
Select commit Hold shift + click to select a range
a7ed1d7
Update .npmrc to get latest python
Nov 9, 2019
e98061c
Merge branch 'master' of https://github.com/Microsoft/vscode-python i…
Nov 20, 2019
bd33869
Update to latest analysis and fix gather icon
Nov 21, 2019
b90e5d2
Use sliceLastExecution and fix
Nov 21, 2019
1754acd
Merge remote-tracking branch 'origin/master' into dev/jim/use-new-gather
Nov 21, 2019
2dd4a52
Merge remote-tracking branch 'origin/master' into dev/jim/use-new-gather
Nov 21, 2019
2bf3d58
Fix import of python analysis
Nov 27, 2019
36b3802
Merge remote-tracking branch 'origin/master' into dev/jim/use-new-gather
Nov 27, 2019
2892421
Merge remote-tracking branch 'origin/master' into dev/jim/use-new-gather
Nov 27, 2019
699f6ec
Update to python-analysis 0.4.12
Dec 7, 2019
6d5d33d
Merge branch 'master' of https://github.com/Microsoft/vscode-python i…
Dec 7, 2019
54053c6
Merge remote-tracking branch 'origin' into dev/jim/use-new-gather
Dec 9, 2019
e73ec59
get gathered text from cell.textSlice
Dec 11, 2019
6d14755
Updates to fix gather logging
Dec 14, 2019
34b0690
Merge branch 'master' into dev/jim/use-new-gather
Dec 16, 2019
6e87570
All updates to using new nb owned gather.
Dec 16, 2019
7f010f8
Merge branch 'master' into dev/jim/use-new-gather
Dec 17, 2019
9282dcf
Merge branch 'master' of https://github.com/Microsoft/vscode-python i…
Dec 19, 2019
4f5a895
Merge branch 'master' into dev/jim/use-new-gather
Jan 8, 2020
39d19f7
Merge branch 'master' into dev/jim/use-new-gather
Jan 14, 2020
811fb97
Working cell execution count codelenses.
Jan 18, 2020
9ca9c62
Merge branch 'master' into dev/jim/use-new-gather
Jan 18, 2020
c69d0c2
Rename gather* to match existing classes
Jan 18, 2020
cb7d0a7
Update to latest python analysis
Jan 21, 2020
7a7c773
lock python analysis ver
Jan 21, 2020
e8bfeef
Merge branch 'master' into dev/jim/use-new-gather
Jan 21, 2020
b8b403b
Merge branch 'master' into dev/jim/use-new-gather
Jan 24, 2020
fe2c87f
Fix MockJupyterNotebook
Jan 24, 2020
a65f35e
Fix casing of getgatherProvider
Jan 24, 2020
7d27f8d
Private Gather support #1
Jan 25, 2020
62a95f6
Merge branch 'master' into dev/jim/use-new-gather
Jan 28, 2020
9c6bd2a
Ensure cellHashProvider can be found
Jan 28, 2020
f77f29a
Revert back to uri for NBExecutionActivated
Jan 28, 2020
645345a
Merge branch 'master' into dev/jim/use-new-gather
Jan 28, 2020
076ff71
Merge branch 'master' into dev/jim/use-new-gather
Jan 29, 2020
c860990
Update new jupyterServerWrapper
Jan 29, 2020
7bda33f
Workaround fix for screwed up cellhashes
Jan 29, 2020
33cc77c
Gracefully fail if python-pgm-analysis is absent
Feb 1, 2020
40eff5c
Merge branch 'master' into dev/jim/use-new-gather
Feb 1, 2020
987340f
Resolve obvious unit test problems.
Feb 1, 2020
c70c3c0
Fixes to make sure that gather.ts is loadable.
Feb 1, 2020
355f725
Merge branch 'master' into dev/jim/use-new-gather
Feb 11, 2020
9453b1a
Merge branch 'master' into dev/jim/use-new-gather
Feb 25, 2020
3f23305
Merge branch 'master' into dev/jim/use-new-gather
Feb 27, 2020
faf985f
Optional gather build, editing, running works.
Feb 27, 2020
ff2cedb
Remove unecessary getAll of IGatherProviders.
Feb 27, 2020
438a94f
Merge branch 'master' into dev/jim/use-new-gather
Feb 27, 2020
cbfe92e
Code cleanup
Feb 27, 2020
d97ef68
Fix gather notebook header
Feb 29, 2020
d8f6f18
Gather icon not only on hover
Feb 29, 2020
d7e3619
Enable gather only in insiders
Feb 29, 2020
99aeb0b
Init gather only if enabled.
Feb 29, 2020
3fab0e1
Add link to survey
Mar 2, 2020
8cf38e0
Merge branch 'master' into dev/jim/use-new-gather
Mar 2, 2020
77e0b1f
Fix spacing in initialization.yml
Mar 2, 2020
cef4a87
Make linter and prettier happy
Mar 2, 2020
d95d441
Make webpack happy with regard to gather
Mar 2, 2020
9760a1d
Don't define ENABLE_GATHER env var
Mar 2, 2020
97a1a98
Couple minor issues found in test
Mar 4, 2020
8ae73d9
Fix a few tests
Mar 4, 2020
51ab887
Merge branch 'master' into dev/jim/use-new-gather
Mar 6, 2020
8da69c9
Everything but dataScienceIocContainer.ts
Mar 6, 2020
eab106b
More fixes
Mar 6, 2020
e7da45a
Whoops, fix IW functional test back to original
Mar 6, 2020
4c43493
Temporarily include datascienceioccontainer in 120
Mar 6, 2020
681fc56
Make linter finally happy.
Mar 6, 2020
815c3cf
Fix Gather functional tests
Mar 6, 2020
3d5135e
A bit more cleanup
Mar 6, 2020
9c1b91b
React to PR feedback
Mar 6, 2020
89517a4
Couple more PR review changes.
Mar 7, 2020
e7d864d
Merge branch 'master' into dev/jim/use-new-gather
Mar 7, 2020
5467f3f
Tweak
Mar 7, 2020
535bb53
More minor cleanup
Mar 7, 2020
43af8d7
Merge branch 'dev/jim/use-new-gather' of https://github.com/microsoft…
Mar 7, 2020
cf41937
Fix provider finders.
Mar 7, 2020
63e0175
Actually fix the provider get problem.
Mar 8, 2020
2f72957
Fix smoke test
Mar 8, 2020
1de83b9
Fix unit test
Mar 8, 2020
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .npmrc
Original file line number Diff line number Diff line change
@@ -1 +1 @@
@types:registry=https://registry.npmjs.org
@types:registry=https://registry.npmjs.org
Comment thread
greazer marked this conversation as resolved.
Resolved
6 changes: 6 additions & 0 deletions .prettierrc.js
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,12 @@ module.exports = {
options: {
tabWidth: 2
}
},
{
files: ['**/datascience/serviceRegistry.ts'],
Comment thread
greazer marked this conversation as resolved.
Resolved
options: {
printWidth: 240
}
}
]
};
12 changes: 11 additions & 1 deletion build/ci/templates/steps/initialization.yml
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ parameters:
compile: 'true'
sqlite: 'false'
installVSCEorNPX: 'true'
enableGather: 'false'

steps:
- bash: |
Expand Down Expand Up @@ -92,6 +93,15 @@ steps:
verbose: true
customCommand: ci

- task: Npm@1
displayName: "Install optional python-program-analysis"
condition: and(succeeded(), eq('${{ parameters.enableGather }}', 'true'))
inputs:
workingDir: ${{ parameters.workingDirectory }}
command: custom
verbose: true
customCommand: "install @msrvida/python-program-analysis"

# On Mac, the command `node` doesn't always point to the current node version.
# Debugger tests use the variable process.env.NODE_PATH
- script: |
Expand All @@ -101,7 +111,7 @@ steps:
displayName: "Setup NODE_PATH for extension (Debugger Tests)"
condition: and(succeeded(), eq(variables['agent.os'], 'Darwin'))

# Install vsce
# Install vsce
- bash: |
npm install -g vsce
displayName: "Install vsce"
Expand Down
8 changes: 7 additions & 1 deletion build/webpack/webpack.extension.config.js
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,9 @@ const configFileName = path.join(constants.ExtensionRootDir, 'tsconfig.extension
const existingModulesInOutDir = common.getListOfExistingModulesInOutDir();
// tslint:disable-next-line:no-var-requires no-require-imports
const FileManagerPlugin = require('filemanager-webpack-plugin');
// If ENABLE_GATHER variable is defined, don't exclude the python-program-analysis pacakge.
// See externals, below.
const ppaPackageList = process.env.ENABLE_GATHER ? [] : ['@msrvida/python-program-analysis'];
const config = {
mode: 'production',
target: 'node',
Expand Down Expand Up @@ -56,7 +59,10 @@ const config = {
{ enforce: 'post', test: /linebreak[\/\\]src[\/\\]linebreaker.js/, loader: 'transform-loader?brfs' }
]
},
externals: ['vscode', 'commonjs', ...existingModulesInOutDir],
// Packages listed in externals keeps webpack from trying to package them.
// The ppaPackageList variable is set to non-empty if the build pipeline has been
//authenticated to install @msrvida/python-program-analysis.
externals: ['vscode', 'commonjs', ...ppaPackageList, ...existingModulesInOutDir],
plugins: [
...common.getDefaultPlugins('extension'),
// Copy pdfkit bits after extension builds. webpack can't handle pdfkit.
Expand Down
5 changes: 0 additions & 5 deletions package-lock.json

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

5 changes: 2 additions & 3 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -1833,13 +1833,13 @@
"python.dataScience.enableGather": {
"type": "boolean",
"default": false,
"description": "Enable code gather for executed cells. For a gathered cell, that cell and only the code it depends on will be exported to a new notebook.",
"description": "Enable experimental code gathering for executed cells. For a gathered cell, that cell and only the code it depends on will be exported to a new notebook.",
"scope": "resource"
},
"python.dataScience.gatherToScript": {
"type": "boolean",
"default": true,
"description": "Gather code to a python script rather than a notebook.",
"description": "If experimental code gather is enabled, gather code to a python script rather than a notebook.",
"scope": "resource"
},
"python.dataScience.codeLenses": {
Expand Down Expand Up @@ -2868,7 +2868,6 @@
"@jupyterlab/services": "^4.2.0",
"@koa/cors": "^3.0.0",
"@loadable/component": "^5.12.0",
"@msrvida/python-program-analysis": "^0.4.1",
"ansi-regex": "^4.1.0",
"arch": "^2.1.0",
"azure-storage": "^2.10.3",
Expand Down
2 changes: 1 addition & 1 deletion package.nls.json
Original file line number Diff line number Diff line change
Expand Up @@ -418,7 +418,7 @@
"DataScience.findJupyterCommandProgressCheckInterpreter": "Checking {0}.",
"DataScience.findJupyterCommandProgressSearchCurrentPath": "Searching current path.",
"DataScience.gatheredScriptDescription": "# This file contains only the code required to produce the results of the gathered cell.\n",
"DataScience.gatheredNotebookDescriptionInMarkdown": "# Gathered Notebook\nGenerated from ```{0}```\n\nThis notebook contains only the code and cells required to produce the same results as the gathered cell.\n\nPlease note that the python analysis is quite conservative, so if it is unsure whether a line of code is necessary for execution, it will err on the side of including it.",
"DataScience.gatheredNotebookDescriptionInMarkdown": "## Gathered Notebook\nGenerated from ```{0}```\n\nThis notebook contains only the code and cells required to produce the same results as the gathered cell.\n\nPlease note that the python analysis is quite conservative, so if it is unsure whether a line of code is necessary for execution, it will err on the side of including it.\n\nAs this is an experimental feature, please let us know how well Gather works for you at [https://aka.ms/gathersurvey](https://aka.ms/gathersurvey)",
"DataScience.savePngTitle": "Save Image",
"DataScience.jupyterSelectURIQuickPickTitle": "Pick how to connect to Jupyter",
"DataScience.jupyterSelectURIQuickPickPlaceholder": "Choose an option",
Expand Down
2 changes: 1 addition & 1 deletion src/client/common/utils/localize.ts
Original file line number Diff line number Diff line change
Expand Up @@ -726,7 +726,7 @@ export namespace DataScience {
);
export const gatheredNotebookDescriptionInMarkdown = localize(
'DataScience.gatheredNotebookDescriptionInMarkdown',
'# Gathered Notebook\nGenerated from ```{0}```\n\nThis notebook contains only the code and cells required to produce the same results as the gathered cell.\n\nPlease note that the python analysis is quite conservative, so if it is unsure whether a line of code is necessary for execution, it will err on the side of including it.'
'## Gathered Notebook\nGenerated from ```{0}```\n\nThis notebook contains only the code and cells required to produce the same results as the gathered cell.\n\nPlease note that the python analysis is quite conservative, so if it is unsure whether a line of code is necessary for execution, it will err on the side of including it.\n\nAs this is an experimental feature, please let us know how well Gather works for you at [https://aka.ms/gathersurvey](https://aka.ms/gathersurvey)'
);
export const savePngTitle = localize('DataScience.savePngTitle', 'Save Image');
export const fallbackToUseActiveInterpeterAsKernel = localize(
Expand Down
48 changes: 48 additions & 0 deletions src/client/datascience/editor-integration/cellhashLogger.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.
'use strict';
import { inject, injectable } from 'inversify';

import { traceError } from '../../common/logger';
import { noop } from '../../common/utils/misc';
import { Identifiers } from '../constants';
import { ICell, ICellHashLogger, ICellHashProvider } from '../types';
import { CellHashProvider } from './cellhashprovider';

// This class provides hashes for debugging jupyter cells. Call getHashes just before starting debugging to compute all of the
// hashes for cells.
@injectable()
export class CellHashLogger implements ICellHashLogger {
constructor(@inject(ICellHashProvider) private provider: ICellHashProvider) {}

public async preExecute(cell: ICell, silent: boolean): Promise<void> {
const providerObj: CellHashProvider = this.provider as CellHashProvider;

try {
if (!silent) {
// Don't log empty cells
const stripped = providerObj.extractExecutableLines(cell);
if (stripped.length > 0 && stripped.find(s => s.trim().length > 0)) {
// When the user adds new code, we know the execution count is increasing
providerObj.incExecutionCount();

// Skip hash on unknown file though
if (cell.file !== Identifiers.EmptyFileName) {
await providerObj.addCellHash(cell, providerObj.getExecutionCount());
}
}
}
} catch (exc) {
// Don't let exceptions in a preExecute mess up normal operation
traceError(exc);
Comment thread
greazer marked this conversation as resolved.
Resolved
}
}

public async postExecute(_cell: ICell, _silent: boolean): Promise<void> {
noop();
}

public getCellHashProvider(): ICellHashProvider {
return this.provider;
}
}
117 changes: 62 additions & 55 deletions src/client/datascience/editor-integration/cellhashprovider.ts
Original file line number Diff line number Diff line change
Expand Up @@ -21,8 +21,7 @@ import {
ICellHashListener,
ICellHashProvider,
IFileHashes,
IInteractiveWindowListener,
INotebookExecutionLogger
IInteractiveWindowListener
} from '../types';

interface IRangedCellHash extends ICellHash {
Expand All @@ -36,16 +35,16 @@ interface IRangedCellHash extends ICellHash {
// This class provides hashes for debugging jupyter cells. Call getHashes just before starting debugging to compute all of the
// hashes for cells.
@injectable()
export class CellHashProvider implements ICellHashProvider, IInteractiveWindowListener, INotebookExecutionLogger {
export class CellHashProvider implements ICellHashProvider, IInteractiveWindowListener {
// tslint:disable-next-line: no-any
private postEmitter: EventEmitter<{ message: string; payload: any }> = new EventEmitter<{
message: string;
// tslint:disable-next-line: no-any
payload: any;
}>();
// Map of file to Map of start line to actual hash
private hashes: Map<string, IRangedCellHash[]> = new Map<string, IRangedCellHash[]>();
private executionCount: number = 0;
private hashes: Map<string, IRangedCellHash[]> = new Map<string, IRangedCellHash[]>();
private updateEventEmitter: EventEmitter<void> = new EventEmitter<void>();

constructor(
Expand Down Expand Up @@ -127,19 +126,7 @@ export class CellHashProvider implements ICellHashProvider, IInteractiveWindowLi
noop();
}

private onChangedDocument(e: TextDocumentChangeEvent) {
// See if the document is in our list of docs to watch
const perFile = this.hashes.get(e.document.fileName);
if (perFile) {
// Apply the content changes to the file's cells.
const docText = e.document.getText();
e.contentChanges.forEach(c => {
this.handleContentChange(docText, c, perFile);
});
}
}

private extractExecutableLines(cell: ICell): string[] {
public extractExecutableLines(cell: ICell): string[] {
const cellMatcher = new CellMatcher(this.configService.getSettings(getCellResource(cell)).datascience);
const lines = splitMultilineString(cell.data.source);
// Only strip this off the first line. Otherwise we want the markers in the code.
Expand All @@ -149,44 +136,7 @@ export class CellHashProvider implements ICellHashProvider, IInteractiveWindowLi
return lines;
}

private handleContentChange(docText: string, c: TextDocumentContentChangeEvent, hashes: IRangedCellHash[]) {
// First compute the number of lines that changed
const lineDiff = c.range.start.line - c.range.end.line + c.text.split('\n').length - 1;
const offsetDiff = c.text.length - c.rangeLength;

// Compute the inclusive offset that is changed by the cell.
const endChangedOffset = c.rangeLength <= 0 ? c.rangeOffset : c.rangeOffset + c.rangeLength - 1;

hashes.forEach(h => {
// See how this existing cell compares to the change
if (h.endOffset < c.rangeOffset) {
// No change. This cell is entirely before the change
} else if (h.startOffset > endChangedOffset) {
// This cell is after the text that got replaced. Adjust its start/end lines
h.line += lineDiff;
h.endLine += lineDiff;
h.startOffset += offsetDiff;
h.endOffset += offsetDiff;
} else if (h.startOffset === endChangedOffset) {
// Cell intersects but exactly, might be a replacement or an insertion
if (h.deleted || c.rangeLength > 0 || lineDiff === 0) {
// Replacement
h.deleted = docText.substr(h.startOffset, h.endOffset - h.startOffset) !== h.realCode;
} else {
// Insertion
h.line += lineDiff;
h.endLine += lineDiff;
h.startOffset += offsetDiff;
h.endOffset += offsetDiff;
}
} else {
// Intersection, delete if necessary
h.deleted = docText.substr(h.startOffset, h.endOffset - h.startOffset) !== h.realCode;
}
});
}

private async addCellHash(cell: ICell, expectedCount: number): Promise<void> {
public async addCellHash(cell: ICell, expectedCount: number): Promise<void> {
// Find the text document that matches. We need more information than
// the add code gives us
const doc = this.documentManager.textDocuments.find(d => this.fileSystem.arePathsSame(d.fileName, cell.file));
Expand Down Expand Up @@ -284,6 +234,63 @@ export class CellHashProvider implements ICellHashProvider, IInteractiveWindowLi
}
}

public getExecutionCount(): number {
return this.executionCount;
}

public incExecutionCount(): void {
this.executionCount += 1;
}

private onChangedDocument(e: TextDocumentChangeEvent) {
// See if the document is in our list of docs to watch
const perFile = this.hashes.get(e.document.fileName);
if (perFile) {
// Apply the content changes to the file's cells.
const docText = e.document.getText();
e.contentChanges.forEach(c => {
this.handleContentChange(docText, c, perFile);
});
}
}

private handleContentChange(docText: string, c: TextDocumentContentChangeEvent, hashes: IRangedCellHash[]) {
// First compute the number of lines that changed
const lineDiff = c.range.start.line - c.range.end.line + c.text.split('\n').length - 1;
const offsetDiff = c.text.length - c.rangeLength;

// Compute the inclusive offset that is changed by the cell.
const endChangedOffset = c.rangeLength <= 0 ? c.rangeOffset : c.rangeOffset + c.rangeLength - 1;

hashes.forEach(h => {
// See how this existing cell compares to the change
if (h.endOffset < c.rangeOffset) {
// No change. This cell is entirely before the change
} else if (h.startOffset > endChangedOffset) {
// This cell is after the text that got replaced. Adjust its start/end lines
h.line += lineDiff;
h.endLine += lineDiff;
h.startOffset += offsetDiff;
h.endOffset += offsetDiff;
} else if (h.startOffset === endChangedOffset) {
// Cell intersects but exactly, might be a replacement or an insertion
if (h.deleted || c.rangeLength > 0 || lineDiff === 0) {
// Replacement
h.deleted = docText.substr(h.startOffset, h.endOffset - h.startOffset) !== h.realCode;
} else {
// Insertion
h.line += lineDiff;
h.endLine += lineDiff;
h.startOffset += offsetDiff;
h.endOffset += offsetDiff;
}
} else {
// Intersection, delete if necessary
h.deleted = docText.substr(h.startOffset, h.endOffset - h.startOffset) !== h.realCode;
}
});
}

private adjustRuntimeForDebugging(
cell: ICell,
source: string[],
Expand Down
Loading