From 0c4523516ca2b810bfb435475e078aba615a4326 Mon Sep 17 00:00:00 2001 From: Ian Huff Date: Mon, 28 Oct 2019 16:08:18 -0700 Subject: [PATCH 1/2] Don't use temp file for directory change --- news/2 Fixes/7567.md | 1 + .../interactive-ipynb/nativeEditor.ts | 14 ++-- .../datascience/jupyter/jupyterImporter.ts | 7 +- src/client/datascience/types.ts | 2 +- .../nativeEditor.unit.test.ts | 67 +++++++++++++++---- .../nativeEditorProvider.unit.test.ts | 10 +-- ...eractiveWindowCommandListener.unit.test.ts | 20 +----- src/test/datascience/mockStatusProvider.ts | 18 +++++ 8 files changed, 93 insertions(+), 46 deletions(-) create mode 100644 news/2 Fixes/7567.md create mode 100644 src/test/datascience/mockStatusProvider.ts diff --git a/news/2 Fixes/7567.md b/news/2 Fixes/7567.md new file mode 100644 index 000000000000..4e44c44ae022 --- /dev/null +++ b/news/2 Fixes/7567.md @@ -0,0 +1 @@ +When exporting a notebook editor to python script don't use the temp file location for generating the export. \ No newline at end of file diff --git a/src/client/datascience/interactive-ipynb/nativeEditor.ts b/src/client/datascience/interactive-ipynb/nativeEditor.ts index 0fd244724c16..8d212c8d04d9 100644 --- a/src/client/datascience/interactive-ipynb/nativeEditor.ts +++ b/src/client/datascience/interactive-ipynb/nativeEditor.ts @@ -467,10 +467,10 @@ export class NativeEditor extends InteractiveBase implements INotebookEditor { * @private * @memberof NativeEditor */ - private async updateVersionInfoInNotebook(): Promise{ + private async updateVersionInfoInNotebook(): Promise { // Use the active interpreter const usableInterpreter = await this.jupyterExecution.getUsableJupyterPython(); - if (usableInterpreter && usableInterpreter.version && this.notebookJson.metadata && this.notebookJson.metadata.language_info){ + if (usableInterpreter && usableInterpreter.version && this.notebookJson.metadata && this.notebookJson.metadata.language_info) { this.notebookJson.metadata.language_info.version = `${usableInterpreter.version.major}.${usableInterpreter.version.minor}.${usableInterpreter.version.patch}`; } } @@ -580,11 +580,11 @@ export class NativeEditor extends InteractiveBase implements INotebookEditor { * @memberof NativeEditor */ private async getStoredContents(): Promise { - const data = this.globalStorage.get<{contents?: string; lastModifiedTimeMs?: number}>(this.getStorageKey()); + const data = this.globalStorage.get<{ contents?: string; lastModifiedTimeMs?: number }>(this.getStorageKey()); // Check whether the file has been modified since the last time the contents were saved. - if (data && data.lastModifiedTimeMs && !this.isUntitled && this.file.scheme === 'file'){ + if (data && data.lastModifiedTimeMs && !this.isUntitled && this.file.scheme === 'file') { const stat = await this.fileSystem.stat(this.file.fsPath); - if (stat.mtime > data.lastModifiedTimeMs){ + if (stat.mtime > data.lastModifiedTimeMs) { return; } } @@ -605,7 +605,7 @@ export class NativeEditor extends InteractiveBase implements INotebookEditor { const key = this.getStorageKey(); // Keep track of the time when this data was saved. // This way when we retrieve the data we can compare it against last modified date of the file. - await this.globalStorage.update(key, {contents, lastModifiedTimeMs: Date.now()}); + await this.globalStorage.update(key, { contents, lastModifiedTimeMs: Date.now() }); } private async close(): Promise { @@ -767,7 +767,7 @@ export class NativeEditor extends InteractiveBase implements INotebookEditor { await this.fileSystem.writeFile(tempFile.filePath, this.generateNotebookContent(cells), { encoding: 'utf-8' }); // Import this file and show it - const contents = await this.importer.importFromFile(tempFile.filePath); + const contents = await this.importer.importFromFile(tempFile.filePath, this.file.fsPath); if (contents) { await this.viewDocument(contents); } diff --git a/src/client/datascience/jupyter/jupyterImporter.ts b/src/client/datascience/jupyter/jupyterImporter.ts index 2fcf89670d2f..2757cde4548b 100644 --- a/src/client/datascience/jupyter/jupyterImporter.ts +++ b/src/client/datascience/jupyter/jupyterImporter.ts @@ -47,19 +47,20 @@ export class JupyterImporter implements INotebookImporter { this.templatePromise = this.createTemplateFile(); } - public async importFromFile(file: string): Promise { + public async importFromFile(contentsFile: string, originalFile?: string): Promise { const template = await this.templatePromise; // If the user has requested it, add a cd command to the imported file so that relative paths still work const settings = this.configuration.getSettings(); let directoryChange: string | undefined; if (settings.datascience.changeDirOnImportExport) { - directoryChange = await this.calculateDirectoryChange(file); + // If an original file is passed in, then use that for calculating the directory change as contents might be an invalid location + directoryChange = await this.calculateDirectoryChange(originalFile ? originalFile : contentsFile); } // Use the jupyter nbconvert functionality to turn the notebook into a python file if (await this.jupyterExecution.isImportSupported()) { - let fileOutput: string = await this.jupyterExecution.importNotebook(file, template); + let fileOutput: string = await this.jupyterExecution.importNotebook(contentsFile, template); if (fileOutput.includes('get_ipython()')) { fileOutput = this.addIPythonImport(fileOutput); } diff --git a/src/client/datascience/types.ts b/src/client/datascience/types.ts index 187452885686..c0c513c76b3c 100644 --- a/src/client/datascience/types.ts +++ b/src/client/datascience/types.ts @@ -186,7 +186,7 @@ export interface IJupyterKernelSpec extends IAsyncDisposable { export const INotebookImporter = Symbol('INotebookImporter'); export interface INotebookImporter extends Disposable { - importFromFile(file: string): Promise; + importFromFile(contentsFile: string, originalFile?: string): Promise; // originalFile is the base file if file is a temp file / location importCellsFromFile(file: string): Promise; importCells(json: string): Promise; } diff --git a/src/test/datascience/interactive-ipynb/nativeEditor.unit.test.ts b/src/test/datascience/interactive-ipynb/nativeEditor.unit.test.ts index f95127b1ca6f..1df0c57192a5 100644 --- a/src/test/datascience/interactive-ipynb/nativeEditor.unit.test.ts +++ b/src/test/datascience/interactive-ipynb/nativeEditor.unit.test.ts @@ -37,7 +37,6 @@ import { JupyterExecutionFactory } from '../../../client/datascience/jupyter/jup import { JupyterExporter } from '../../../client/datascience/jupyter/jupyterExporter'; import { JupyterImporter } from '../../../client/datascience/jupyter/jupyterImporter'; import { JupyterVariables } from '../../../client/datascience/jupyter/jupyterVariables'; -import { StatusProvider } from '../../../client/datascience/statusProvider'; import { ThemeFinder } from '../../../client/datascience/themeFinder'; import { ICodeCssGenerator, @@ -49,7 +48,6 @@ import { INotebookEditorProvider, INotebookExporter, INotebookImporter, - IStatusProvider, IThemeFinder } from '../../../client/datascience/types'; import { IInterpreterService } from '../../../client/interpreter/contracts'; @@ -58,6 +56,7 @@ import { createEmptyCell } from '../../../datascience-ui/interactive-common/main import { waitForCondition } from '../../common'; import { noop } from '../../core'; import { MockMemento } from '../../mocks/mementos'; +import { MockStatusProvider } from '../mockStatusProvider'; // tslint:disable: no-any chai-vague-errors no-unused-expression @@ -66,7 +65,7 @@ suite('Data Science - Native Editor', () => { let workspace: IWorkspaceService; let configService: IConfigurationService; let fileSystem: IFileSystem; - let doctManager: IDocumentManager; + let docManager: IDocumentManager; let dsErrorHandler: IDataScienceErrorHandler; let cmdManager: ICommandManager; let liveShare: ILiveShareApi; @@ -76,7 +75,7 @@ suite('Data Science - Native Editor', () => { const disposables: Disposable[] = []; let cssGenerator: ICodeCssGenerator; let themeFinder: IThemeFinder; - let statusProvider: IStatusProvider; + let statusProvider: MockStatusProvider; let executionProvider: IJupyterExecution; let exportProvider: INotebookExporter; let editorProvider: INotebookEditorProvider; @@ -188,7 +187,7 @@ suite('Data Science - Native Editor', () => { storageUpdateSpy = sinon.spy(storage, 'update'); configService = mock(ConfigurationService); fileSystem = mock(FileSystem); - doctManager = mock(DocumentManager); + docManager = mock(DocumentManager); dsErrorHandler = mock(DataScienceErrorHandler); cmdManager = mock(CommandManager); workspace = mock(WorkspaceService); @@ -198,7 +197,7 @@ suite('Data Science - Native Editor', () => { webPanelProvider = mock(WebPanelProvider); cssGenerator = mock(CodeCssGenerator); themeFinder = mock(ThemeFinder); - statusProvider = mock(StatusProvider); + statusProvider = new MockStatusProvider(); executionProvider = mock(JupyterExecutionFactory); exportProvider = mock(JupyterExporter); editorProvider = mock(NativeEditorProvider); @@ -218,10 +217,11 @@ suite('Data Science - Native Editor', () => { when(interpreterService.onDidChangeInterpreter).thenReturn(interprerterChangeEvent.event); const editorChangeEvent = new EventEmitter(); - when(doctManager.onDidChangeActiveTextEditor).thenReturn(editorChangeEvent.event); + when(docManager.onDidChangeActiveTextEditor).thenReturn(editorChangeEvent.event); const sessionChangedEvent = new EventEmitter(); when(executionProvider.sessionChanged).thenReturn(sessionChangedEvent.event); + }); teardown(() => { @@ -234,13 +234,13 @@ suite('Data Science - Native Editor', () => { [], instance(liveShare), instance(applicationShell), - instance(doctManager), + instance(docManager), instance(interpreterService), instance(webPanelProvider), disposables, instance(cssGenerator), instance(themeFinder), - instance(statusProvider), + statusProvider, instance(executionProvider), instance(fileSystem), instance(configService), @@ -418,7 +418,7 @@ suite('Data Science - Native Editor', () => { expect(newEditor.cells).to.be.lengthOf(3); }); - test('Pyton version info will be updated in notebook when a cell has been executed', async () => { + test('Python version info will be updated in notebook when a cell has been executed', async () => { const file = Uri.parse('file://foo.ipynb'); const editor = createEditor(); @@ -429,11 +429,11 @@ suite('Data Science - Native Editor', () => { expect(contents.metadata!.language_info!.version).to.not.equal('10.11.12'); // When a cell is executed, then ensure we store the python version info in the notebook data. - const version: Version = {build: [], major: 10, minor: 11, patch: 12, prerelease: [], raw: '10.11.12'}; - when(executionProvider.getUsableJupyterPython()).thenResolve(({version} as any)); + const version: Version = { build: [], major: 10, minor: 11, patch: 12, prerelease: [], raw: '10.11.12' }; + when(executionProvider.getUsableJupyterPython()).thenResolve(({ version } as any)); try { - editor.onMessage(InteractiveWindowMessages.SubmitNewCell, {code: 'hello', id: '1'}); + editor.onMessage(InteractiveWindowMessages.SubmitNewCell, { code: 'hello', id: '1' }); } catch { // Ignore errors related to running cells, assume that works. noop(); @@ -453,4 +453,45 @@ suite('Data Science - Native Editor', () => { contents = JSON.parse(editor.contents) as nbformat.INotebookContent; expect(contents.metadata!.language_info!.version).to.equal('10.11.12'); }); + + test('Export to Python script file from notebook.', async () => { + // Temp file location needed for export + const tempFile = { + dispose: () => { + return undefined; + }, + filePath: '/foo/bar.ipynb' + }; + when(fileSystem.createTemporaryFile('.ipynb')).thenResolve(tempFile); + + // Set up our importer to return file contents, check that we have the correct temp file location and + // original file location + when(importer.importFromFile('/foo/bar.ipynb', '\\foo.ipynb')).thenResolve('# File Contents'); + + // Just return empty objects here, we don't care about open or show function, just that they were called + when(docManager.openTextDocument({ language: 'python', content: '# File Contents' })).thenResolve({} as any); + when(docManager.showTextDocument(anything(), anything())).thenResolve({} as any); + + const file = Uri.parse('file:///foo.ipynb'); + + const editor = createEditor(); + await editor.load(baseFile, file); + expect(editor.contents).to.be.equal(baseFile); + + // Make our call to actually export + editor.onMessage(InteractiveWindowMessages.Export, editor.cells); + + await waitForCondition(async () => { + try { + // Wait until showTextDocument has been called, that's the signal that export is done + verify(docManager.showTextDocument(anything(), anything())).atLeast(1); + return true; + } catch { + return false; + } + }, 1_000, 'Timeout'); + + // Verify that we also opened our text document not exact match as verify doesn't seem to match that + verify(docManager.openTextDocument(anything())).once(); + }); }); diff --git a/src/test/datascience/interactive-ipynb/nativeEditorProvider.unit.test.ts b/src/test/datascience/interactive-ipynb/nativeEditorProvider.unit.test.ts index 96ce959e9f2d..0bd37ec7514d 100644 --- a/src/test/datascience/interactive-ipynb/nativeEditorProvider.unit.test.ts +++ b/src/test/datascience/interactive-ipynb/nativeEditorProvider.unit.test.ts @@ -30,7 +30,7 @@ suite('Data Science - Native Editor Provider', () => { let workspace: IWorkspaceService; let configService: IConfigurationService; let fileSystem: IFileSystem; - let doctManager: IDocumentManager; + let docManager: IDocumentManager; let dsErrorHandler: IDataScienceErrorHandler; let cmdManager: ICommandManager; let svcContainer: IServiceContainer; @@ -39,7 +39,7 @@ suite('Data Science - Native Editor Provider', () => { svcContainer = mock(ServiceContainer); configService = mock(ConfigurationService); fileSystem = mock(FileSystem); - doctManager = mock(DocumentManager); + docManager = mock(DocumentManager); dsErrorHandler = mock(DataScienceErrorHandler); cmdManager = mock(CommandManager); workspace = mock(WorkspaceService); @@ -53,7 +53,7 @@ suite('Data Science - Native Editor Provider', () => { instance(workspace), instance(configService), instance(fileSystem), - instance(doctManager), + instance(docManager), instance(cmdManager), instance(dsErrorHandler) ); @@ -74,8 +74,8 @@ suite('Data Science - Native Editor Provider', () => { const eventEmitter = new EventEmitter(); const editor = typemoq.Mock.ofType(); when(configService.getSettings()).thenReturn({ datascience: { useNotebookEditor: true } } as any); - when(doctManager.onDidChangeActiveTextEditor).thenReturn(eventEmitter.event); - when(doctManager.visibleTextEditors).thenReturn([]); + when(docManager.onDidChangeActiveTextEditor).thenReturn(eventEmitter.event); + when(docManager.visibleTextEditors).thenReturn([]); editor.setup(e => e.closed).returns(() => new EventEmitter().event); editor.setup(e => e.executed).returns(() => new EventEmitter().event); editor.setup(e => (e as any).then).returns(() => undefined); diff --git a/src/test/datascience/interactiveWindowCommandListener.unit.test.ts b/src/test/datascience/interactiveWindowCommandListener.unit.test.ts index 648708b8f7e1..5309aa3a74c5 100644 --- a/src/test/datascience/interactiveWindowCommandListener.unit.test.ts +++ b/src/test/datascience/interactiveWindowCommandListener.unit.test.ts @@ -7,7 +7,7 @@ import { anything, instance, mock, when } from 'ts-mockito'; import { Matcher } from 'ts-mockito/lib/matcher/type/Matcher'; import * as TypeMoq from 'typemoq'; import * as uuid from 'uuid/v4'; -import { Disposable, EventEmitter, Uri } from 'vscode'; +import { EventEmitter, Uri } from 'vscode'; import { ApplicationShell } from '../../client/common/application/applicationShell'; import { PythonSettings } from '../../client/common/configSettings'; @@ -27,11 +27,9 @@ import { JupyterExecutionFactory } from '../../client/datascience/jupyter/jupyte import { JupyterExporter } from '../../client/datascience/jupyter/jupyterExporter'; import { JupyterImporter } from '../../client/datascience/jupyter/jupyterImporter'; import { - IInteractiveBase, IInteractiveWindow, INotebook, - INotebookServer, - IStatusProvider + INotebookServer } from '../../client/datascience/types'; import { InterpreterService } from '../../client/interpreter/interpreterService'; import { KnownSearchPathsForInterpreters } from '../../client/interpreter/locators/services/KnownPathsService'; @@ -41,6 +39,7 @@ import { MockAutoSelectionService } from '../mocks/autoSelector'; import * as vscodeMocks from '../vscode-mock'; import { MockCommandManager } from './mockCommandManager'; import { MockDocumentManager } from './mockDocumentManager'; +import { MockStatusProvider } from './mockStatusProvider'; // tslint:disable:no-any no-http-string no-multiline-string max-func-body-length @@ -53,19 +52,6 @@ function createTypeMoq(tag: string): TypeMoq.IMock { return result; } -class MockStatusProvider implements IStatusProvider { - public set(_message: string, _timeout?: number, _cancel?: () => void, _panel?: IInteractiveBase): Disposable { - return { - dispose: noop - }; - } - - public waitWithStatus(promise: () => Promise, _message: string, _timeout?: number, _canceled?: () => void, _panel?: IInteractiveBase): Promise { - return promise(); - } - -} - // tslint:disable:no-any no-http-string no-multiline-string max-func-body-length suite('Interactive window command listener', async () => { const interpreterService = mock(InterpreterService); diff --git a/src/test/datascience/mockStatusProvider.ts b/src/test/datascience/mockStatusProvider.ts new file mode 100644 index 000000000000..1d658e2fabc8 --- /dev/null +++ b/src/test/datascience/mockStatusProvider.ts @@ -0,0 +1,18 @@ +import { Disposable } from 'vscode'; +import { + IInteractiveBase, + IStatusProvider +} from '../../client/datascience/types'; +import { noop } from '../core'; +export class MockStatusProvider implements IStatusProvider { + public set(_message: string, _timeout?: number, _cancel?: () => void, _panel?: IInteractiveBase): Disposable { + return { + dispose: noop + }; + } + + public waitWithStatus(promise: () => Promise, _message: string, _timeout?: number, _canceled?: () => void, _panel?: IInteractiveBase): Promise { + return promise(); + } + +} From 9c3bffdbf7ab440534222b1eff93bc8ce2cfbacd Mon Sep 17 00:00:00 2001 From: Ian Huff Date: Tue, 29 Oct 2019 08:57:06 -0700 Subject: [PATCH 2/2] fix test to work on mac / linux --- .../datascience/interactive-ipynb/nativeEditor.unit.test.ts | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/test/datascience/interactive-ipynb/nativeEditor.unit.test.ts b/src/test/datascience/interactive-ipynb/nativeEditor.unit.test.ts index 1df0c57192a5..4753973f4f76 100644 --- a/src/test/datascience/interactive-ipynb/nativeEditor.unit.test.ts +++ b/src/test/datascience/interactive-ipynb/nativeEditor.unit.test.ts @@ -466,14 +466,13 @@ suite('Data Science - Native Editor', () => { // Set up our importer to return file contents, check that we have the correct temp file location and // original file location - when(importer.importFromFile('/foo/bar.ipynb', '\\foo.ipynb')).thenResolve('# File Contents'); + const file = Uri.parse('file:///foo.ipynb'); + when(importer.importFromFile('/foo/bar.ipynb', file.fsPath)).thenResolve('# File Contents'); // Just return empty objects here, we don't care about open or show function, just that they were called when(docManager.openTextDocument({ language: 'python', content: '# File Contents' })).thenResolve({} as any); when(docManager.showTextDocument(anything(), anything())).thenResolve({} as any); - const file = Uri.parse('file:///foo.ipynb'); - const editor = createEditor(); await editor.load(baseFile, file); expect(editor.contents).to.be.equal(baseFile);