From 704135af5c5365baf462619627abf3e882299f0b Mon Sep 17 00:00:00 2001 From: Jim Griesmer Date: Thu, 24 Oct 2019 17:29:07 -0700 Subject: [PATCH 01/42] Main changes for update --- build/ci/postInstall.js | 2 +- package-lock.json | 531 ++---------------- package.json | 2 +- src/client/datascience/common.ts | 2 +- .../editor-integration/cellhashprovider.ts | 2 +- src/client/datascience/gather/gather.ts | 46 +- .../interactive-ipynb/nativeEditor.ts | 2 +- .../datascience/jupyter/jupyterDebugger.ts | 2 +- .../datascience/jupyter/jupyterExporter.ts | 2 +- .../datascience/jupyter/jupyterImporter.ts | 2 +- .../datascience/jupyter/jupyterNotebook.ts | 2 +- .../datascience/jupyter/jupyterVariables.ts | 2 +- src/client/datascience/types.ts | 2 +- .../history-react/interactiveCell.tsx | 2 +- .../interactive-common/cellInput.tsx | 2 +- .../interactive-common/cellOutput.tsx | 2 +- .../interactive-common/mainState.ts | 2 +- .../native-editor/nativeCell.tsx | 2 +- .../dataviewer.functional.test.tsx | 2 +- .../datascience/gather/gather.unit.test.ts | 2 +- ...eractiveWindowCommandListener.unit.test.ts | 2 +- .../datascience/jupyterVariables.unit.test.ts | 2 +- src/test/datascience/mockJupyterManager.ts | 2 +- src/test/datascience/mockJupyterRequest.ts | 2 +- .../datascience/notebook.functional.test.ts | 2 +- 25 files changed, 85 insertions(+), 538 deletions(-) diff --git a/build/ci/postInstall.js b/build/ci/postInstall.js index 42f85b47fc46..8a26aab09d7a 100644 --- a/build/ci/postInstall.js +++ b/build/ci/postInstall.js @@ -14,7 +14,7 @@ var constants_1 = require("../constants"); * The solution is to modify the type definition file after `npm install`. */ function fixJupyterLabDTSFiles() { - var relativePath = path.join('node_modules', '@jupyterlab', 'coreutils', 'lib', 'settingregistry.d.ts'); + var relativePath = path.join('node_modules', '@jupyterlab', 'services', 'node_modules', 'coreutils', 'lb', 'settingregistry.d.ts'); var filePath = path.join(constants_1.ExtensionRootDir, relativePath); if (!fs.existsSync(filePath)) { throw new Error("Type Definition file from JupyterLab not found '" + filePath + "' (pvsc post install script)"); diff --git a/package-lock.json b/package-lock.json index 5abaca6979da..86d010556993 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1197,204 +1197,6 @@ "integrity": "sha512-EzRFg92bRSD1W/zeuNkeGwph0nkWf+pP2l/lYW4/5hav7RjKKBN5kV1Ix7Tvi0CMu3pC4Wi/U7rNisiJMR3ORg==", "dev": true }, - "@jupyterlab/apputils": { - "version": "0.19.1", - "resolved": "https://registry.npmjs.org/@jupyterlab/apputils/-/apputils-0.19.1.tgz", - "integrity": "sha512-//vajDyVyKwXU7qycRBJC37dljjWeya+YpJV3z5sSXgVBefEBZ0kcrJ92ugDSht4I4SRv6x+kw6K9mBXKaYu9Q==", - "requires": { - "@jupyterlab/coreutils": "^2.2.1", - "@jupyterlab/services": "^3.2.1", - "@phosphor/algorithm": "^1.1.2", - "@phosphor/commands": "^1.6.1", - "@phosphor/coreutils": "^1.3.0", - "@phosphor/disposable": "^1.1.2", - "@phosphor/domutils": "^1.1.2", - "@phosphor/messaging": "^1.2.2", - "@phosphor/properties": "^1.1.2", - "@phosphor/signaling": "^1.2.2", - "@phosphor/virtualdom": "^1.1.2", - "@phosphor/widgets": "^1.6.0", - "@types/react": "~16.4.13", - "react": "~16.4.2", - "react-dom": "~16.4.2", - "sanitize-html": "~1.18.2" - }, - "dependencies": { - "@types/react": { - "version": "16.4.18", - "resolved": "https://registry.npmjs.org/@types/react/-/react-16.4.18.tgz", - "integrity": "sha512-eFzJKEg6pdeaukVLVZ8Xb79CTl/ysX+ExmOfAAqcFlCCK5TgFDD9kWR0S18sglQ3EmM8U+80enjUqbfnUyqpdA==", - "requires": { - "@types/prop-types": "*", - "csstype": "^2.2.0" - } - }, - "react": { - "version": "16.4.2", - "resolved": "https://registry.npmjs.org/react/-/react-16.4.2.tgz", - "integrity": "sha512-dMv7YrbxO4y2aqnvA7f/ik9ibeLSHQJTI6TrYAenPSaQ6OXfb+Oti+oJiy8WBxgRzlKatYqtCjphTgDSCEiWFg==", - "requires": { - "fbjs": "^0.8.16", - "loose-envify": "^1.1.0", - "object-assign": "^4.1.1", - "prop-types": "^15.6.0" - } - }, - "react-dom": { - "version": "16.4.2", - "resolved": "https://registry.npmjs.org/react-dom/-/react-dom-16.4.2.tgz", - "integrity": "sha512-Usl73nQqzvmJN+89r97zmeUpQDKDlh58eX6Hbs/ERdDHzeBzWy+ENk7fsGQ+5KxArV1iOFPT46/VneklK9zoWw==", - "requires": { - "fbjs": "^0.8.16", - "loose-envify": "^1.1.0", - "object-assign": "^4.1.1", - "prop-types": "^15.6.0" - } - } - } - }, - "@jupyterlab/attachments": { - "version": "0.19.1", - "resolved": "https://registry.npmjs.org/@jupyterlab/attachments/-/attachments-0.19.1.tgz", - "integrity": "sha512-JujH1ExOao6ytb0J305iqOtj+GxISq/9zgt9kugwUCYZpvGdHau7WAYQs5YDPIQSZjSdYzjBREssFoirOOSVmA==", - "requires": { - "@jupyterlab/coreutils": "^2.2.1", - "@jupyterlab/observables": "^2.1.1", - "@jupyterlab/rendermime": "^0.19.1", - "@jupyterlab/rendermime-interfaces": "^1.2.1", - "@phosphor/disposable": "^1.1.2", - "@phosphor/signaling": "^1.2.2" - } - }, - "@jupyterlab/cells": { - "version": "0.19.1", - "resolved": "https://registry.npmjs.org/@jupyterlab/cells/-/cells-0.19.1.tgz", - "integrity": "sha512-7vVriCTVC6sXPDvh1/L2WzdAga2Xkg0cFgEcXuIJLwt25ppmqs9DYgMhhUBr0G734mZwmQDXCJ/BqOUw1qgHKg==", - "requires": { - "@jupyterlab/apputils": "^0.19.1", - "@jupyterlab/attachments": "^0.19.1", - "@jupyterlab/codeeditor": "^0.19.1", - "@jupyterlab/codemirror": "^0.19.1", - "@jupyterlab/coreutils": "^2.2.1", - "@jupyterlab/observables": "^2.1.1", - "@jupyterlab/outputarea": "^0.19.1", - "@jupyterlab/rendermime": "^0.19.1", - "@jupyterlab/services": "^3.2.1", - "@phosphor/coreutils": "^1.3.0", - "@phosphor/messaging": "^1.2.2", - "@phosphor/signaling": "^1.2.2", - "@phosphor/widgets": "^1.6.0", - "react": "~16.4.2" - }, - "dependencies": { - "react": { - "version": "16.4.2", - "resolved": "https://registry.npmjs.org/react/-/react-16.4.2.tgz", - "integrity": "sha512-dMv7YrbxO4y2aqnvA7f/ik9ibeLSHQJTI6TrYAenPSaQ6OXfb+Oti+oJiy8WBxgRzlKatYqtCjphTgDSCEiWFg==", - "requires": { - "fbjs": "^0.8.16", - "loose-envify": "^1.1.0", - "object-assign": "^4.1.1", - "prop-types": "^15.6.0" - } - } - } - }, - "@jupyterlab/codeeditor": { - "version": "0.19.1", - "resolved": "https://registry.npmjs.org/@jupyterlab/codeeditor/-/codeeditor-0.19.1.tgz", - "integrity": "sha512-mCL4YiCCX5JOAIs21mleSmlkejAw6FVLwmvKGxBCwoNj+cSYUKzGBYuA2xvxAZi/5HaiQU8R+ITbAwk2QoMc6w==", - "requires": { - "@jupyterlab/coreutils": "^2.2.1", - "@jupyterlab/observables": "^2.1.1", - "@phosphor/coreutils": "^1.3.0", - "@phosphor/disposable": "^1.1.2", - "@phosphor/messaging": "^1.2.2", - "@phosphor/signaling": "^1.2.2", - "@phosphor/widgets": "^1.6.0", - "react": "~16.4.2", - "react-dom": "~16.4.2" - }, - "dependencies": { - "react": { - "version": "16.4.2", - "resolved": "https://registry.npmjs.org/react/-/react-16.4.2.tgz", - "integrity": "sha512-dMv7YrbxO4y2aqnvA7f/ik9ibeLSHQJTI6TrYAenPSaQ6OXfb+Oti+oJiy8WBxgRzlKatYqtCjphTgDSCEiWFg==", - "requires": { - "fbjs": "^0.8.16", - "loose-envify": "^1.1.0", - "object-assign": "^4.1.1", - "prop-types": "^15.6.0" - } - }, - "react-dom": { - "version": "16.4.2", - "resolved": "https://registry.npmjs.org/react-dom/-/react-dom-16.4.2.tgz", - "integrity": "sha512-Usl73nQqzvmJN+89r97zmeUpQDKDlh58eX6Hbs/ERdDHzeBzWy+ENk7fsGQ+5KxArV1iOFPT46/VneklK9zoWw==", - "requires": { - "fbjs": "^0.8.16", - "loose-envify": "^1.1.0", - "object-assign": "^4.1.1", - "prop-types": "^15.6.0" - } - } - } - }, - "@jupyterlab/codemirror": { - "version": "0.19.1", - "resolved": "https://registry.npmjs.org/@jupyterlab/codemirror/-/codemirror-0.19.1.tgz", - "integrity": "sha512-JbZRm9vW1lN4Y4VghBQEys7vWoaZM54E0OdTlWjJq1kF8WhKWzp8Do/tnQ5fKVUGw40BMB0xBnfAaHHCSs5vHw==", - "requires": { - "@jupyterlab/apputils": "^0.19.1", - "@jupyterlab/codeeditor": "^0.19.1", - "@jupyterlab/coreutils": "^2.2.1", - "@jupyterlab/observables": "^2.1.1", - "@phosphor/algorithm": "^1.1.2", - "@phosphor/coreutils": "^1.3.0", - "@phosphor/disposable": "^1.1.2", - "@phosphor/signaling": "^1.2.2", - "codemirror": "~5.39.0" - } - }, - "@jupyterlab/coreutils": { - "version": "2.2.1", - "resolved": "https://registry.npmjs.org/@jupyterlab/coreutils/-/coreutils-2.2.1.tgz", - "integrity": "sha512-XkGMBXqEAnENC4fK/L3uEqqxyNUtf4TI/1XNDln7d5VOPHQJSBTbYlBAZ0AQotn2qbs4WqmV6icxje2ITVedqQ==", - "requires": { - "@phosphor/algorithm": "^1.1.2", - "@phosphor/coreutils": "^1.3.0", - "@phosphor/disposable": "^1.1.2", - "@phosphor/signaling": "^1.2.2", - "ajv": "~5.1.6", - "comment-json": "^1.1.3", - "minimist": "~1.2.0", - "moment": "~2.21.0", - "path-posix": "~1.0.0", - "url-parse": "~1.4.3" - }, - "dependencies": { - "ajv": { - "version": "5.1.6", - "resolved": "https://registry.npmjs.org/ajv/-/ajv-5.1.6.tgz", - "integrity": "sha1-Sy8aGd7Ok9V6whYDfj6XkcfdFWQ=", - "requires": { - "co": "^4.6.0", - "json-schema-traverse": "^0.3.0", - "json-stable-stringify": "^1.0.1" - } - }, - "json-schema-traverse": { - "version": "0.3.1", - "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.3.1.tgz", - "integrity": "sha1-NJptRMU6Ud6JtAgFxdXlm0F9M0A=" - }, - "moment": { - "version": "2.21.0", - "resolved": "https://registry.npmjs.org/moment/-/moment-2.21.0.tgz", - "integrity": "sha512-TCZ36BjURTeFTM/CwRcViQlfkMvL1/vFISuNLO5GkcVm1+QHfbSiNqZuWeMFjj1/3+uAjXswgRk30j1kkLYJBQ==" - } - } - }, "@jupyterlab/observables": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/@jupyterlab/observables/-/observables-2.2.0.tgz", @@ -1407,54 +1209,6 @@ "@phosphor/signaling": "^1.2.3" } }, - "@jupyterlab/outputarea": { - "version": "0.19.2", - "resolved": "https://registry.npmjs.org/@jupyterlab/outputarea/-/outputarea-0.19.2.tgz", - "integrity": "sha512-uYL/RfwY//83UWsmENg4b4s6wXr9OWE4zbKtMY4zoJoT5tsnWyIiSnvy26m/Pslo1Y1wlLjVPqCoJFcEARKb+A==", - "requires": { - "@jupyterlab/apputils": "^0.19.1", - "@jupyterlab/coreutils": "^2.2.1", - "@jupyterlab/observables": "^2.1.1", - "@jupyterlab/rendermime": "^0.19.1", - "@jupyterlab/rendermime-interfaces": "^1.2.1", - "@jupyterlab/services": "^3.2.1", - "@phosphor/algorithm": "^1.1.2", - "@phosphor/coreutils": "^1.3.0", - "@phosphor/disposable": "^1.1.2", - "@phosphor/messaging": "^1.2.2", - "@phosphor/signaling": "^1.2.2", - "@phosphor/widgets": "^1.6.0" - } - }, - "@jupyterlab/rendermime": { - "version": "0.19.1", - "resolved": "https://registry.npmjs.org/@jupyterlab/rendermime/-/rendermime-0.19.1.tgz", - "integrity": "sha512-MD+/lMwFa/b9QtJyFghTXFqkuRha74+vWTaPcfzREdHRiKYHYPHWOD/gb8cLIr4c7J3VaEVZWEpI8udYgyANvA==", - "requires": { - "@jupyterlab/apputils": "^0.19.1", - "@jupyterlab/codemirror": "^0.19.1", - "@jupyterlab/coreutils": "^2.2.1", - "@jupyterlab/observables": "^2.1.1", - "@jupyterlab/rendermime-interfaces": "^1.2.1", - "@jupyterlab/services": "^3.2.1", - "@phosphor/algorithm": "^1.1.2", - "@phosphor/coreutils": "^1.3.0", - "@phosphor/messaging": "^1.2.2", - "@phosphor/signaling": "^1.2.2", - "@phosphor/widgets": "^1.6.0", - "ansi_up": "^3.0.0", - "marked": "~0.4.0" - } - }, - "@jupyterlab/rendermime-interfaces": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/@jupyterlab/rendermime-interfaces/-/rendermime-interfaces-1.4.0.tgz", - "integrity": "sha512-6GtbudtK7Yl37ldnQUQ6hTUDzjVgemiDXokEdsSTLPOPoPwKw3gaEBfxrOz/LBITfqIfbpuBzYU8lQ4rHq0TfQ==", - "requires": { - "@phosphor/coreutils": "^1.3.1", - "@phosphor/widgets": "^1.8.0" - } - }, "@jupyterlab/services": { "version": "3.2.1", "resolved": "https://registry.npmjs.org/@jupyterlab/services/-/services-3.2.1.tgz", @@ -1517,15 +1271,9 @@ } }, "@msrvida/python-program-analysis": { - "version": "0.2.6", - "resolved": "https://registry.npmjs.org/@msrvida/python-program-analysis/-/python-program-analysis-0.2.6.tgz", - "integrity": "sha512-DlclIdsuwMktplZlxhTCky0fLIAy925MCEWKqdU/6PLV6L+FiG7jZ7/V2qZohjmBdJEuJoifERWzkUrP+PaH0A==", - "requires": { - "@jupyterlab/cells": "^0.19.1", - "@jupyterlab/coreutils": "^2.2.1", - "@jupyterlab/rendermime": "^0.19.1", - "js-yaml": "^3.13.1" - } + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/@msrvida/python-program-analysis/-/python-program-analysis-0.4.0.tgz", + "integrity": "sha512-/bqastZynDio4DSmFQsRBiXlLl7FBrKnyUgJqZoRHw6Uzs+LF3AHMkt1NzdRn0qauBMX5G/wIzgDqL1pgSt4LA==" }, "@nteract/markdown": { "version": "3.0.1", @@ -1687,43 +1435,6 @@ "@phosphor/algorithm": "^1.1.3" } }, - "@phosphor/commands": { - "version": "1.7.1", - "resolved": "https://registry.npmjs.org/@phosphor/commands/-/commands-1.7.1.tgz", - "integrity": "sha512-KELPYLrNLVkMA5XntDogQkKXWbhLhpjxLBD75faywoe4GCyVsm//CA7Wn50+eVo0pI87z27Qbtzo0TR6NH4Jvw==", - "requires": { - "@phosphor/algorithm": "^1.2.0", - "@phosphor/coreutils": "^1.3.1", - "@phosphor/disposable": "^1.3.0", - "@phosphor/domutils": "^1.1.4", - "@phosphor/keyboard": "^1.1.3", - "@phosphor/signaling": "^1.3.0" - }, - "dependencies": { - "@phosphor/algorithm": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/@phosphor/algorithm/-/algorithm-1.2.0.tgz", - "integrity": "sha512-C9+dnjXyU2QAkWCW6QVDGExk4hhwxzAKf5/FIuYlHAI9X5vFv99PYm0EREDxX1PbMuvfFBZhPNu0PvuSDQ7sFA==" - }, - "@phosphor/disposable": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/@phosphor/disposable/-/disposable-1.3.0.tgz", - "integrity": "sha512-wHQov7HoS20mU6yuEz5ZMPhfxHdcxGovjPoid0QwccUEOm33UBkWlxaJGm9ONycezIX8je7ZuPOf/gf7JI6Dlg==", - "requires": { - "@phosphor/algorithm": "^1.2.0", - "@phosphor/signaling": "^1.3.0" - } - }, - "@phosphor/signaling": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/@phosphor/signaling/-/signaling-1.3.0.tgz", - "integrity": "sha512-ZbG2Mof4LGSkaEuDicqA2o2TKu3i5zanjr2GkevI/82aKBD7cI1NGLGT55HZwtE87/gOF4FIM3d3DeyrFDMjMQ==", - "requires": { - "@phosphor/algorithm": "^1.2.0" - } - } - } - }, "@phosphor/coreutils": { "version": "1.3.1", "resolved": "https://registry.npmjs.org/@phosphor/coreutils/-/coreutils-1.3.1.tgz", @@ -1738,49 +1449,6 @@ "@phosphor/signaling": "^1.2.3" } }, - "@phosphor/domutils": { - "version": "1.1.4", - "resolved": "https://registry.npmjs.org/@phosphor/domutils/-/domutils-1.1.4.tgz", - "integrity": "sha512-ivwq5TWjQpKcHKXO8PrMl+/cKqbgxPClPiCKc1gwbMd+6hnW5VLwNG0WBzJTxCzXK43HxX18oH+tOZ3E04wc3w==" - }, - "@phosphor/dragdrop": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/@phosphor/dragdrop/-/dragdrop-1.4.0.tgz", - "integrity": "sha512-JqmDAKczviUe7NEkiDf/A6H2glgVmHAREip8dGBli4lvV+CQqPFyl4Xm7XCnR9qiEqNrP+0SfwPpywNa0me3nQ==", - "requires": { - "@phosphor/coreutils": "^1.3.1", - "@phosphor/disposable": "^1.3.0" - }, - "dependencies": { - "@phosphor/algorithm": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/@phosphor/algorithm/-/algorithm-1.2.0.tgz", - "integrity": "sha512-C9+dnjXyU2QAkWCW6QVDGExk4hhwxzAKf5/FIuYlHAI9X5vFv99PYm0EREDxX1PbMuvfFBZhPNu0PvuSDQ7sFA==" - }, - "@phosphor/disposable": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/@phosphor/disposable/-/disposable-1.3.0.tgz", - "integrity": "sha512-wHQov7HoS20mU6yuEz5ZMPhfxHdcxGovjPoid0QwccUEOm33UBkWlxaJGm9ONycezIX8je7ZuPOf/gf7JI6Dlg==", - "requires": { - "@phosphor/algorithm": "^1.2.0", - "@phosphor/signaling": "^1.3.0" - } - }, - "@phosphor/signaling": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/@phosphor/signaling/-/signaling-1.3.0.tgz", - "integrity": "sha512-ZbG2Mof4LGSkaEuDicqA2o2TKu3i5zanjr2GkevI/82aKBD7cI1NGLGT55HZwtE87/gOF4FIM3d3DeyrFDMjMQ==", - "requires": { - "@phosphor/algorithm": "^1.2.0" - } - } - } - }, - "@phosphor/keyboard": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/@phosphor/keyboard/-/keyboard-1.1.3.tgz", - "integrity": "sha512-dzxC/PyHiD6mXaESRy6PZTd9JeK+diwG1pyngkyUf127IXOEzubTIbu52VSdpGBklszu33ws05BAGDa4oBE4mQ==" - }, "@phosphor/messaging": { "version": "1.2.3", "resolved": "https://registry.npmjs.org/@phosphor/messaging/-/messaging-1.2.3.tgz", @@ -1790,11 +1458,6 @@ "@phosphor/collections": "^1.1.3" } }, - "@phosphor/properties": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/@phosphor/properties/-/properties-1.1.3.tgz", - "integrity": "sha512-GiglqzU77s6+tFVt6zPq9uuyu/PLQPFcqZt914ZhJ4cN/7yNI/SLyMzpYZ56IRMXvzK9TUgbRna6URE3XAwFUg==" - }, "@phosphor/signaling": { "version": "1.2.3", "resolved": "https://registry.npmjs.org/@phosphor/signaling/-/signaling-1.2.3.tgz", @@ -1803,80 +1466,6 @@ "@phosphor/algorithm": "^1.1.3" } }, - "@phosphor/virtualdom": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/@phosphor/virtualdom/-/virtualdom-1.2.0.tgz", - "integrity": "sha512-L9mKNhK2XtVjzjuHLG2uYuepSz8uPyu6vhF4EgCP0rt0TiLYaZeHwuNu3XeFbul9DMOn49eBpye/tfQVd4Ks+w==", - "requires": { - "@phosphor/algorithm": "^1.2.0" - }, - "dependencies": { - "@phosphor/algorithm": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/@phosphor/algorithm/-/algorithm-1.2.0.tgz", - "integrity": "sha512-C9+dnjXyU2QAkWCW6QVDGExk4hhwxzAKf5/FIuYlHAI9X5vFv99PYm0EREDxX1PbMuvfFBZhPNu0PvuSDQ7sFA==" - } - } - }, - "@phosphor/widgets": { - "version": "1.9.2", - "resolved": "https://registry.npmjs.org/@phosphor/widgets/-/widgets-1.9.2.tgz", - "integrity": "sha512-93BOdp3lGsEdkpa+bv+XhIKqRyNKStu71IeDtvqiRyRyMjDnMjwfQCjNDdYbsWYyWWKk4TH7buC0cIlmbIPAHQ==", - "requires": { - "@phosphor/algorithm": "^1.2.0", - "@phosphor/commands": "^1.7.1", - "@phosphor/coreutils": "^1.3.1", - "@phosphor/disposable": "^1.3.0", - "@phosphor/domutils": "^1.1.4", - "@phosphor/dragdrop": "^1.4.0", - "@phosphor/keyboard": "^1.1.3", - "@phosphor/messaging": "^1.3.0", - "@phosphor/properties": "^1.1.3", - "@phosphor/signaling": "^1.3.0", - "@phosphor/virtualdom": "^1.2.0" - }, - "dependencies": { - "@phosphor/algorithm": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/@phosphor/algorithm/-/algorithm-1.2.0.tgz", - "integrity": "sha512-C9+dnjXyU2QAkWCW6QVDGExk4hhwxzAKf5/FIuYlHAI9X5vFv99PYm0EREDxX1PbMuvfFBZhPNu0PvuSDQ7sFA==" - }, - "@phosphor/collections": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/@phosphor/collections/-/collections-1.2.0.tgz", - "integrity": "sha512-T9/0EjSuY6+ga2LIFRZ0xupciOR3Qnyy8Q95lhGTC0FXZUFwC8fl9e8On6IcwasCszS+1n8dtZUWSIynfgdpzw==", - "requires": { - "@phosphor/algorithm": "^1.2.0" - } - }, - "@phosphor/disposable": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/@phosphor/disposable/-/disposable-1.3.0.tgz", - "integrity": "sha512-wHQov7HoS20mU6yuEz5ZMPhfxHdcxGovjPoid0QwccUEOm33UBkWlxaJGm9ONycezIX8je7ZuPOf/gf7JI6Dlg==", - "requires": { - "@phosphor/algorithm": "^1.2.0", - "@phosphor/signaling": "^1.3.0" - } - }, - "@phosphor/messaging": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/@phosphor/messaging/-/messaging-1.3.0.tgz", - "integrity": "sha512-k0JE+BTMKlkM335S2AmmJxoYYNRwOdW5jKBqLgjJdGRvUQkM0+2i60ahM45+J23atGJDv9esKUUBINiKHFhLew==", - "requires": { - "@phosphor/algorithm": "^1.2.0", - "@phosphor/collections": "^1.2.0" - } - }, - "@phosphor/signaling": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/@phosphor/signaling/-/signaling-1.3.0.tgz", - "integrity": "sha512-ZbG2Mof4LGSkaEuDicqA2o2TKu3i5zanjr2GkevI/82aKBD7cI1NGLGT55HZwtE87/gOF4FIM3d3DeyrFDMjMQ==", - "requires": { - "@phosphor/algorithm": "^1.2.0" - } - } - } - }, "@sindresorhus/is": { "version": "0.7.0", "resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-0.7.0.tgz", @@ -2248,7 +1837,8 @@ "@types/prop-types": { "version": "15.7.1", "resolved": "https://registry.npmjs.org/@types/prop-types/-/prop-types-15.7.1.tgz", - "integrity": "sha512-CFzn9idOEpHrgdw8JsoTkaDDyRWk1jrzIV8djzcgpq0y9tG4B4lFT+Nxh52DVpDXV+n4+NPNv7M1Dj5uMp6XFg==" + "integrity": "sha512-CFzn9idOEpHrgdw8JsoTkaDDyRWk1jrzIV8djzcgpq0y9tG4B4lFT+Nxh52DVpDXV+n4+NPNv7M1Dj5uMp6XFg==", + "dev": true }, "@types/react": { "version": "16.8.23", @@ -2866,11 +2456,6 @@ "integrity": "sha1-qCJQ3bABXponyoLoLqYDu/pF768=", "dev": true }, - "ansi_up": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/ansi_up/-/ansi_up-3.0.0.tgz", - "integrity": "sha1-J/Rdj0V9nO/1nk6gPI5vE8GjA+g=" - }, "anymatch": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/anymatch/-/anymatch-2.0.0.tgz", @@ -3285,7 +2870,8 @@ "array-uniq": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/array-uniq/-/array-uniq-1.0.3.tgz", - "integrity": "sha1-r2rId6Jcx/dOBYiUdThY39sk/bY=" + "integrity": "sha1-r2rId6Jcx/dOBYiUdThY39sk/bY=", + "dev": true }, "array-unique": { "version": "0.3.2", @@ -3323,7 +2909,8 @@ "asap": { "version": "2.0.6", "resolved": "https://registry.npmjs.org/asap/-/asap-2.0.6.tgz", - "integrity": "sha1-5QNHYR1+aQlDIIu9r+vLwvuGbUY=" + "integrity": "sha1-5QNHYR1+aQlDIIu9r+vLwvuGbUY=", + "dev": true }, "asn1": { "version": "0.2.4", @@ -5626,11 +5213,6 @@ "urlgrey": "^0.4.4" } }, - "codemirror": { - "version": "5.39.2", - "resolved": "https://registry.npmjs.org/codemirror/-/codemirror-5.39.2.tgz", - "integrity": "sha512-mchBy0kQ1Wggi+e58SmoLgKO4nG7s/BqNg6/6TRbhsnXI/KRG+fKAvRQ1LLhZZ6ZtUoDQ0dl5aMhE+IkSRh60Q==" - }, "collapse-white-space": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/collapse-white-space/-/collapse-white-space-1.0.5.tgz", @@ -6057,7 +5639,8 @@ "core-js": { "version": "1.2.7", "resolved": "https://registry.npmjs.org/core-js/-/core-js-1.2.7.tgz", - "integrity": "sha1-ZSKUwUZR2yj6k70tX/KYOk8IxjY=" + "integrity": "sha1-ZSKUwUZR2yj6k70tX/KYOk8IxjY=", + "dev": true }, "core-js-compat": { "version": "3.1.4", @@ -6567,7 +6150,8 @@ "csstype": { "version": "2.6.6", "resolved": "https://registry.npmjs.org/csstype/-/csstype-2.6.6.tgz", - "integrity": "sha512-RpFbQGUE74iyPgvr46U9t1xoQBM8T4BL8SxrN66Le2xYAPSaDJJKeztV3awugusb3g3G9iL8StmkBBXhcbbXhg==" + "integrity": "sha512-RpFbQGUE74iyPgvr46U9t1xoQBM8T4BL8SxrN66Le2xYAPSaDJJKeztV3awugusb3g3G9iL8StmkBBXhcbbXhg==", + "dev": true }, "cucumber-html-reporter": { "version": "4.0.5", @@ -7441,6 +7025,7 @@ "version": "0.1.1", "resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-0.1.1.tgz", "integrity": "sha512-l0IU0pPzLWSHBcieZbpOKgkIn3ts3vAh7ZuFyXNwJxJXk/c4Gwj9xaTJwIDVQCXawWD0qb3IzMGH5rglQaO0XA==", + "dev": true, "requires": { "domelementtype": "^1.3.0", "entities": "^1.1.1" @@ -7466,7 +7051,8 @@ "domelementtype": { "version": "1.3.1", "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-1.3.1.tgz", - "integrity": "sha512-BSKB+TSpMpFI/HOxCNr1O8aMOTZ8hT3pM3GQ0w/mWRmkhEDSFJkkyzz4XQsBV44BChwGkrDfMyjVD0eA2aFV3w==" + "integrity": "sha512-BSKB+TSpMpFI/HOxCNr1O8aMOTZ8hT3pM3GQ0w/mWRmkhEDSFJkkyzz4XQsBV44BChwGkrDfMyjVD0eA2aFV3w==", + "dev": true }, "domexception": { "version": "1.0.1", @@ -7481,6 +7067,7 @@ "version": "2.4.2", "resolved": "https://registry.npmjs.org/domhandler/-/domhandler-2.4.2.tgz", "integrity": "sha512-JiK04h0Ht5u/80fdLMCEmV4zkNh2BcoMFBmZ/91WtYZ8qVXSKjiw7fXMgFPnHcSZgOo3XdinHvmnDUeMf5R4wA==", + "dev": true, "requires": { "domelementtype": "1" } @@ -7489,6 +7076,7 @@ "version": "1.7.0", "resolved": "https://registry.npmjs.org/domutils/-/domutils-1.7.0.tgz", "integrity": "sha512-Lgd2XcJ/NjEw+7tFvfKxOzCYKZsdct5lczQ2ZaQY8Djz7pfAD3Gbp8ySJWtreII/vDlMVmxwa6pHmdxIYgttDg==", + "dev": true, "requires": { "dom-serializer": "0", "domelementtype": "1" @@ -7783,7 +7371,8 @@ "entities": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/entities/-/entities-1.1.2.tgz", - "integrity": "sha512-f2LZMYl1Fzu7YSBKg+RoROelpOaNrcGmE9AZubeDfrCEia483oW4MI4VyFd5VNHIgQ/7qm1I0wUHK1eJnn2y2w==" + "integrity": "sha512-f2LZMYl1Fzu7YSBKg+RoROelpOaNrcGmE9AZubeDfrCEia483oW4MI4VyFd5VNHIgQ/7qm1I0wUHK1eJnn2y2w==", + "dev": true }, "env-paths": { "version": "1.0.0", @@ -8633,6 +8222,7 @@ "version": "0.8.17", "resolved": "https://registry.npmjs.org/fbjs/-/fbjs-0.8.17.tgz", "integrity": "sha1-xNWY6taUkRJlPWWIsBpc3Nn5D90=", + "dev": true, "requires": { "core-js": "^1.0.0", "isomorphic-fetch": "^2.1.1", @@ -10762,6 +10352,7 @@ "version": "3.10.1", "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-3.10.1.tgz", "integrity": "sha512-IgieNijUMbkDovyoKObU1DUhm1iwNYE/fuifEoEHfd1oZKZDaONBSkal7Y01shxsM49R4XaMdGez3WnF9UfiCQ==", + "dev": true, "requires": { "domelementtype": "^1.3.1", "domhandler": "^2.3.0", @@ -11588,6 +11179,7 @@ "version": "2.2.1", "resolved": "https://registry.npmjs.org/isomorphic-fetch/-/isomorphic-fetch-2.2.1.tgz", "integrity": "sha1-YRrhrPFPXoH3KVB0coGf6XM1WKk=", + "dev": true, "requires": { "node-fetch": "^1.0.1", "whatwg-fetch": ">=0.10.0" @@ -12313,11 +11905,6 @@ "integrity": "sha1-soqmKIorn8ZRA1x3EfZathkDMaY=", "dev": true }, - "lodash.clonedeep": { - "version": "4.5.0", - "resolved": "https://registry.npmjs.org/lodash.clonedeep/-/lodash.clonedeep-4.5.0.tgz", - "integrity": "sha1-4j8/nE+Pvd6HJSnBBxhXoIblzO8=" - }, "lodash.curry": { "version": "4.1.1", "resolved": "https://registry.npmjs.org/lodash.curry/-/lodash.curry-4.1.1.tgz", @@ -12342,11 +11929,6 @@ "integrity": "sha1-yQRGkMIeBClL6qUXcS/e0fqI3pg=", "dev": true }, - "lodash.escaperegexp": { - "version": "4.1.2", - "resolved": "https://registry.npmjs.org/lodash.escaperegexp/-/lodash.escaperegexp-4.1.2.tgz", - "integrity": "sha1-ZHYsSGGAglGKw99Mz11YhtriA0c=" - }, "lodash.flatten": { "version": "4.4.0", "resolved": "https://registry.npmjs.org/lodash.flatten/-/lodash.flatten-4.4.0.tgz", @@ -12380,17 +11962,8 @@ "lodash.isplainobject": { "version": "4.0.6", "resolved": "https://registry.npmjs.org/lodash.isplainobject/-/lodash.isplainobject-4.0.6.tgz", - "integrity": "sha1-fFJqUtibRcRcxpC4gWO+BJf1UMs=" - }, - "lodash.isstring": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/lodash.isstring/-/lodash.isstring-4.0.1.tgz", - "integrity": "sha1-1SfftUVuynzJu5XV2ur4i6VKVFE=" - }, - "lodash.mergewith": { - "version": "4.6.2", - "resolved": "https://registry.npmjs.org/lodash.mergewith/-/lodash.mergewith-4.6.2.tgz", - "integrity": "sha512-GK3g5RPZWTRSeLSpgP8Xhra+pnjBC56q9FZYe1d5RN3TJ35dbkGy3YqBSMbyCrlbi+CM9Z3Jk5yTL7RCsqboyQ==" + "integrity": "sha1-fFJqUtibRcRcxpC4gWO+BJf1UMs=", + "dev": true }, "lodash.some": { "version": "4.6.0", @@ -12620,11 +12193,6 @@ "uc.micro": "^1.0.5" } }, - "marked": { - "version": "0.4.0", - "resolved": "https://registry.npmjs.org/marked/-/marked-0.4.0.tgz", - "integrity": "sha512-tMsdNBgOsrUophCAFQl0XPe6Zqk/uy9gnue+jIIKhykO51hxyu6uNx7zBPy0+y/WKYVZZMspV9YeXLNdKk+iYw==" - }, "martinez-polygon-clipping": { "version": "0.1.5", "resolved": "https://registry.npmjs.org/martinez-polygon-clipping/-/martinez-polygon-clipping-0.1.5.tgz", @@ -13933,7 +13501,8 @@ "number-is-nan": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/number-is-nan/-/number-is-nan-1.0.1.tgz", - "integrity": "sha1-CXtgK1NCKlIsGvuHkDGDNpQaAR0=" + "integrity": "sha1-CXtgK1NCKlIsGvuHkDGDNpQaAR0=", + "dev": true }, "numeral": { "version": "2.0.6", @@ -15063,16 +14632,6 @@ "integrity": "sha1-AerA/jta9xoqbAL+q7jB/vfgDqs=", "dev": true }, - "postcss": { - "version": "6.0.23", - "resolved": "https://registry.npmjs.org/postcss/-/postcss-6.0.23.tgz", - "integrity": "sha512-soOk1h6J3VMTZtVeVpv15/Hpdl2cBLX3CAw4TAbkpTJiNPk9YP/zWcD1ND+xEtvyuuvKzbxliTOIyvkSeSJ6ag==", - "requires": { - "chalk": "^2.4.1", - "source-map": "^0.6.1", - "supports-color": "^5.4.0" - } - }, "postcss-modules-extract-imports": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/postcss-modules-extract-imports/-/postcss-modules-extract-imports-1.2.1.tgz", @@ -15307,6 +14866,7 @@ "version": "7.3.1", "resolved": "https://registry.npmjs.org/promise/-/promise-7.3.1.tgz", "integrity": "sha512-nolQXZ/4L+bP/UGlkfaIujX9BKxGwmQ9OT4mOt5yvy8iK1h3wqTEJCijzGANTCCl9nWjY41juyAn2K3Q1hLLTg==", + "dev": true, "requires": { "asap": "~2.0.3" } @@ -16643,23 +16203,6 @@ "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" }, - "sanitize-html": { - "version": "1.18.5", - "resolved": "https://registry.npmjs.org/sanitize-html/-/sanitize-html-1.18.5.tgz", - "integrity": "sha512-z0MV+AqOnDZVSQQHr/vwimRykKVyPuGZnjWDzIiV1mdgQEG9HMx9qrEapcOQeUmSsPvHZ04BXTuXQkB/vvbU9A==", - "requires": { - "chalk": "^2.3.0", - "htmlparser2": "^3.9.0", - "lodash.clonedeep": "^4.5.0", - "lodash.escaperegexp": "^4.1.2", - "lodash.isplainobject": "^4.0.6", - "lodash.isstring": "^4.0.1", - "lodash.mergewith": "^4.6.0", - "postcss": "^6.0.14", - "srcset": "^1.0.0", - "xtend": "^4.0.0" - } - }, "sass-loader": { "version": "7.1.0", "resolved": "https://registry.npmjs.org/sass-loader/-/sass-loader-7.1.0.tgz", @@ -16922,7 +16465,8 @@ "setimmediate": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/setimmediate/-/setimmediate-1.0.5.tgz", - "integrity": "sha1-KQy7Iy4waULX1+qbg3Mqt4VvgoU=" + "integrity": "sha1-KQy7Iy4waULX1+qbg3Mqt4VvgoU=", + "dev": true }, "setprototypeof": { "version": "1.1.1", @@ -17379,15 +16923,6 @@ "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", "integrity": "sha1-BOaSb2YolTVPPdAVIDYzuFcpfiw=" }, - "srcset": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/srcset/-/srcset-1.0.0.tgz", - "integrity": "sha1-pWad4StC87HV6D7QPHEEb8SPQe8=", - "requires": { - "array-uniq": "^1.0.2", - "number-is-nan": "^1.0.0" - } - }, "sshpk": { "version": "1.16.1", "resolved": "https://registry.npmjs.org/sshpk/-/sshpk-1.16.1.tgz", @@ -18918,7 +18453,8 @@ "ua-parser-js": { "version": "0.7.20", "resolved": "https://registry.npmjs.org/ua-parser-js/-/ua-parser-js-0.7.20.tgz", - "integrity": "sha512-8OaIKfzL5cpx8eCMAhhvTlft8GYF8b2eQr6JkCyVdrgjcytyOmPCXrqXFcUnhonRpLlh5yxEZVohm6mzaowUOw==" + "integrity": "sha512-8OaIKfzL5cpx8eCMAhhvTlft8GYF8b2eQr6JkCyVdrgjcytyOmPCXrqXFcUnhonRpLlh5yxEZVohm6mzaowUOw==", + "dev": true }, "uc.micro": { "version": "1.0.6", @@ -20082,7 +19618,8 @@ "whatwg-fetch": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/whatwg-fetch/-/whatwg-fetch-3.0.0.tgz", - "integrity": "sha512-9GSJUgz1D4MfyKU7KRqwOjXCXTqWdFNvEr7eUBYchQiVc744mqK/MzXPNR2WsPkmkOa4ywfg8C2n8h+13Bey1Q==" + "integrity": "sha512-9GSJUgz1D4MfyKU7KRqwOjXCXTqWdFNvEr7eUBYchQiVc744mqK/MzXPNR2WsPkmkOa4ywfg8C2n8h+13Bey1Q==", + "dev": true }, "whatwg-mimetype": { "version": "2.3.0", diff --git a/package.json b/package.json index f7b874406d61..1ef3ca3f5494 100644 --- a/package.json +++ b/package.json @@ -2679,7 +2679,7 @@ "dependencies": { "@blueprintjs/select": "3.10.0", "@jupyterlab/services": "^3.2.1", - "@msrvida/python-program-analysis": "^0.2.6", + "@msrvida/python-program-analysis": "^0.4.1", "ansi-regex": "^4.1.0", "arch": "^2.1.0", "azure-storage": "^2.10.3", diff --git a/src/client/datascience/common.ts b/src/client/datascience/common.ts index fb2b611032a2..5f7092d8a3cd 100644 --- a/src/client/datascience/common.ts +++ b/src/client/datascience/common.ts @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. 'use strict'; -import { nbformat } from '@jupyterlab/coreutils/lib/nbformat'; +import { nbformat } from '@jupyterlab/services/node_modules/@jupyterlab/coreutils/lib/nbformat'; import { noop } from '../../test/core'; diff --git a/src/client/datascience/editor-integration/cellhashprovider.ts b/src/client/datascience/editor-integration/cellhashprovider.ts index ad0c7f210661..c088f441f7fa 100644 --- a/src/client/datascience/editor-integration/cellhashprovider.ts +++ b/src/client/datascience/editor-integration/cellhashprovider.ts @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. 'use strict'; -import { nbformat } from '@jupyterlab/coreutils'; +import { nbformat } from '@jupyterlab/services/node_modules/@jupyterlab/coreutils'; import * as hashjs from 'hash.js'; import { inject, injectable, multiInject, optional } from 'inversify'; import { Event, EventEmitter, Position, Range, TextDocumentChangeEvent, TextDocumentContentChangeEvent } from 'vscode'; diff --git a/src/client/datascience/gather/gather.ts b/src/client/datascience/gather/gather.ts index 8a578989a719..f87955b2f643 100644 --- a/src/client/datascience/gather/gather.ts +++ b/src/client/datascience/gather/gather.ts @@ -1,7 +1,5 @@ -import { DataflowAnalyzer } from '@msrvida/python-program-analysis'; -import { Cell as ICell, LogCell } from '@msrvida/python-program-analysis/dist/es5/cell'; -import { CellSlice } from '@msrvida/python-program-analysis/dist/es5/cellslice'; -import { ExecutionLogSlicer } from '@msrvida/python-program-analysis/dist/es5/log-slicer'; +import { CellSlice, DataflowAnalyzer, ExecutionLogSlicer } from '@msrvida/python-program-analysis'; +import { Cell as IGatherCell } from '@msrvida/python-program-analysis/dist/es5/cell'; import { inject, injectable } from 'inversify'; // tslint:disable-next-line: no-require-imports @@ -23,7 +21,7 @@ import { CellState, ICell as IVscCell, IGatherExecution, INotebookExecutionLogge */ @injectable() export class GatherExecution implements IGatherExecution, INotebookExecutionLogger { - private _executionSlicer: ExecutionLogSlicer; + private _executionSlicer: ExecutionLogSlicer; private dataflowAnalyzer: DataflowAnalyzer; private _enabled: boolean; @@ -63,7 +61,7 @@ export class GatherExecution implements IGatherExecution, INotebookExecutionLogg cloneCell.data.source = cellMatcher.stripFirstMarker(concatMultilineStringInput(vscCell.data.source)); // Convert IVscCell to IGatherCell - const cell = convertVscToGatherCell(cloneCell) as LogCell; + const cell = convertVscToGatherCell(cloneCell) as IGatherCell; // Call internal logging method this._executionSlicer.logExecution(cell); @@ -76,8 +74,8 @@ export class GatherExecution implements IGatherExecution, INotebookExecutionLogg */ public gatherCode(vscCell: IVscCell): string { // sliceAllExecutions does a lookup based on executionEventId - const cell = convertVscToGatherCell(vscCell); - if (cell === undefined) { + const gatherCell = convertVscToGatherCell(vscCell); + if (gatherCell === undefined) { return ''; } @@ -85,12 +83,28 @@ export class GatherExecution implements IGatherExecution, INotebookExecutionLogg const defaultCellMarker = this.configService.getSettings().datascience.defaultCellMarker || Identifiers.DefaultCodeCellMarker; // Call internal slice method - const slices = this._executionSlicer.sliceAllExecutions(cell); + const slices = this._executionSlicer.sliceAllExecutions(gatherCell.persistentId); const program = slices.length > 0 ? slices[0].cellSlices.reduce(concat, '').replace(/#%%/g, defaultCellMarker) : ''; // Add a comment at the top of the file explaining what gather does const descriptor = '# This file contains the minimal amount of code required to produce the code cell you gathered.\n'; return descriptor.concat(program); + + // const slicedExecution: SlicedExecution = this._executionSlicer.sliceAllExecutions(gatherCell.executionEventId).pop(); + // const depCells = this._executionSlicer.getDependentCells(gatherCell.executionEventId); + + // const descriptor = '# This file contains the minimal amount of code required to produce the code cell you gathered.\n'; + // let script: string = `${descriptor}`; + // if (slicedExecution.cellSlices.length > 0) { + // slicedExecution.cellSlices.forEach((c: CellSlice) => { script = script.concat(c.cell.text); }); + // } + + // if (depCells.length > 0) { + // depCells.forEach((c) => { script = script.concat(c.text); }); + // } + // script = script.replace(/#%%/g, defaultCellMarker); + + // return script; } public get executionSlicer() { @@ -122,7 +136,8 @@ export class GatherExecution implements IGatherExecution, INotebookExecutionLogg /** * Accumulator to concatenate cell slices for a sliced program, preserving cell structures. */ -function concat(existingText: string, newText: CellSlice) { +function concat(existingText: string, newText: CellSlice): string { + // tslint:disable-next-line: jsdoc-format // Include our cell marker so that cell slices are preserved return `${existingText}#%%\n${newText.textSliceLines}\n\n`; } @@ -131,14 +146,11 @@ function concat(existingText: string, newText: CellSlice) { * This is called to convert VS Code ICells to Gather ICells for logging. * @param cell A cell object conforming to the VS Code cell interface */ -function convertVscToGatherCell(cell: IVscCell): ICell | undefined { +function convertVscToGatherCell(cell: IVscCell): IGatherCell | undefined { // This should always be true since we only want to log code cells. Putting this here so types match for outputs property if (cell.data.cell_type === 'code') { - const result: ICell = { + const result: IGatherCell = { // tslint:disable-next-line no-unnecessary-local-variable - id: cell.id, - gathered: false, - dirty: false, text: cell.data.source, // This may need to change for native notebook support since in the original Gather code this refers to the number of times that this same cell was executed @@ -147,9 +159,7 @@ function convertVscToGatherCell(cell: IVscCell): ICell | undefined { // This may need to change for native notebook support, since this is intended to persist in the metadata for a notebook that is saved and then re-loaded persistentId: cell.id, - outputs: cell.data.outputs, - hasError: cell.state === CellState.error, - is_cell: true + hasError: cell.state === CellState.error // tslint:disable-next-line: no-any } as any; return result; diff --git a/src/client/datascience/interactive-ipynb/nativeEditor.ts b/src/client/datascience/interactive-ipynb/nativeEditor.ts index 8342e6e70de1..bf37fd64eacc 100644 --- a/src/client/datascience/interactive-ipynb/nativeEditor.ts +++ b/src/client/datascience/interactive-ipynb/nativeEditor.ts @@ -3,7 +3,7 @@ 'use strict'; import '../../common/extensions'; -import { nbformat } from '@jupyterlab/coreutils/lib/nbformat'; +import { nbformat } from '@jupyterlab/services/node_modules/@jupyterlab/coreutils/lib/nbformat'; import * as detectIndent from 'detect-indent'; import { inject, injectable, multiInject, named } from 'inversify'; import * as path from 'path'; diff --git a/src/client/datascience/jupyter/jupyterDebugger.ts b/src/client/datascience/jupyter/jupyterDebugger.ts index 56afb727606e..31001309db0d 100644 --- a/src/client/datascience/jupyter/jupyterDebugger.ts +++ b/src/client/datascience/jupyter/jupyterDebugger.ts @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. 'use strict'; -import { nbformat } from '@jupyterlab/coreutils'; +import { nbformat } from '@jupyterlab/services/node_modules/@jupyterlab/coreutils'; import { inject, injectable } from 'inversify'; import * as uuid from 'uuid/v4'; import { DebugConfiguration } from 'vscode'; diff --git a/src/client/datascience/jupyter/jupyterExporter.ts b/src/client/datascience/jupyter/jupyterExporter.ts index 89fed8dfbbf5..0116ad43898f 100644 --- a/src/client/datascience/jupyter/jupyterExporter.ts +++ b/src/client/datascience/jupyter/jupyterExporter.ts @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. 'use strict'; -import { nbformat } from '@jupyterlab/coreutils'; +import { nbformat } from '@jupyterlab/services/node_modules/@jupyterlab/coreutils'; import { inject, injectable } from 'inversify'; import * as os from 'os'; import * as path from 'path'; diff --git a/src/client/datascience/jupyter/jupyterImporter.ts b/src/client/datascience/jupyter/jupyterImporter.ts index 2fcf89670d2f..b7e5e9a3de5c 100644 --- a/src/client/datascience/jupyter/jupyterImporter.ts +++ b/src/client/datascience/jupyter/jupyterImporter.ts @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. 'use strict'; -import { nbformat } from '@jupyterlab/coreutils/lib/nbformat'; +import { nbformat } from '@jupyterlab/services/node_modules/@jupyterlab/coreutils'; import * as fs from 'fs-extra'; import { inject, injectable } from 'inversify'; import * as os from 'os'; diff --git a/src/client/datascience/jupyter/jupyterNotebook.ts b/src/client/datascience/jupyter/jupyterNotebook.ts index 12dbeac05e87..f8597bda28fc 100644 --- a/src/client/datascience/jupyter/jupyterNotebook.ts +++ b/src/client/datascience/jupyter/jupyterNotebook.ts @@ -3,8 +3,8 @@ 'use strict'; import '../../common/extensions'; -import { nbformat } from '@jupyterlab/coreutils'; import { Kernel, KernelMessage } from '@jupyterlab/services'; +import { nbformat } from '@jupyterlab/services/node_modules/@jupyterlab/coreutils'; import * as fs from 'fs-extra'; import { Observable } from 'rxjs/Observable'; import { Subscriber } from 'rxjs/Subscriber'; diff --git a/src/client/datascience/jupyter/jupyterVariables.ts b/src/client/datascience/jupyter/jupyterVariables.ts index f452ec922877..2923b7736bfc 100644 --- a/src/client/datascience/jupyter/jupyterVariables.ts +++ b/src/client/datascience/jupyter/jupyterVariables.ts @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. 'use strict'; -import { nbformat } from '@jupyterlab/coreutils'; +import { nbformat } from '@jupyterlab/services/node_modules/@jupyterlab/coreutils'; import { JSONObject } from '@phosphor/coreutils'; import { inject, injectable } from 'inversify'; import * as path from 'path'; diff --git a/src/client/datascience/types.ts b/src/client/datascience/types.ts index 187452885686..2027a8fb3e68 100644 --- a/src/client/datascience/types.ts +++ b/src/client/datascience/types.ts @@ -1,8 +1,8 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. 'use strict'; -import { nbformat } from '@jupyterlab/coreutils'; import { Kernel, KernelMessage } from '@jupyterlab/services/lib/kernel'; +import { nbformat } from '@jupyterlab/services/node_modules/@jupyterlab/coreutils'; import { JSONObject } from '@phosphor/coreutils'; import { Observable } from 'rxjs/Observable'; import { diff --git a/src/datascience-ui/history-react/interactiveCell.tsx b/src/datascience-ui/history-react/interactiveCell.tsx index a437b975db2d..473d9f55ca9e 100644 --- a/src/datascience-ui/history-react/interactiveCell.tsx +++ b/src/datascience-ui/history-react/interactiveCell.tsx @@ -3,7 +3,7 @@ 'use strict'; import '../../client/common/extensions'; -import { nbformat } from '@jupyterlab/coreutils'; +import { nbformat } from '@jupyterlab/services/node_modules/@jupyterlab/coreutils'; import * as fastDeepEqual from 'fast-deep-equal'; import * as monacoEditor from 'monaco-editor/esm/vs/editor/editor.api'; import * as React from 'react'; diff --git a/src/datascience-ui/interactive-common/cellInput.tsx b/src/datascience-ui/interactive-common/cellInput.tsx index 41ec875fcd1e..6e65d1ada50d 100644 --- a/src/datascience-ui/interactive-common/cellInput.tsx +++ b/src/datascience-ui/interactive-common/cellInput.tsx @@ -3,7 +3,7 @@ 'use strict'; import '../../client/common/extensions'; -import { nbformat } from '@jupyterlab/coreutils'; +import { nbformat } from '@jupyterlab/services/node_modules/@jupyterlab/coreutils'; import * as monacoEditor from 'monaco-editor/esm/vs/editor/editor.api'; import * as React from 'react'; diff --git a/src/datascience-ui/interactive-common/cellOutput.tsx b/src/datascience-ui/interactive-common/cellOutput.tsx index c3e978f0a670..4c406c254d35 100644 --- a/src/datascience-ui/interactive-common/cellOutput.tsx +++ b/src/datascience-ui/interactive-common/cellOutput.tsx @@ -6,7 +6,7 @@ import '../../client/common/extensions'; // tslint:disable-next-line: no-var-requires no-require-imports const ansiToHtml = require('ansi-to-html'); -import { nbformat } from '@jupyterlab/coreutils'; +import { nbformat } from '@jupyterlab/services/node_modules/@jupyterlab/coreutils'; import { JSONObject } from '@phosphor/coreutils'; import ansiRegex from 'ansi-regex'; // tslint:disable-next-line: no-require-imports diff --git a/src/datascience-ui/interactive-common/mainState.ts b/src/datascience-ui/interactive-common/mainState.ts index 5f1767669169..29d59fb4c026 100644 --- a/src/datascience-ui/interactive-common/mainState.ts +++ b/src/datascience-ui/interactive-common/mainState.ts @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. 'use strict'; -import { nbformat } from '@jupyterlab/coreutils'; +import { nbformat } from '@jupyterlab/services/node_modules/@jupyterlab/coreutils'; // tslint:disable-next-line: no-require-imports import cloneDeep = require('lodash/cloneDeep'); import * as monacoEditor from 'monaco-editor/esm/vs/editor/editor.api'; diff --git a/src/datascience-ui/native-editor/nativeCell.tsx b/src/datascience-ui/native-editor/nativeCell.tsx index 5be34db77cf0..b767d12bca56 100644 --- a/src/datascience-ui/native-editor/nativeCell.tsx +++ b/src/datascience-ui/native-editor/nativeCell.tsx @@ -3,7 +3,7 @@ 'use strict'; import '../../client/common/extensions'; -import { nbformat } from '@jupyterlab/coreutils'; +import { nbformat } from '@jupyterlab/services/node_modules/@jupyterlab/coreutils'; import * as fastDeepEqual from 'fast-deep-equal'; import * as React from 'react'; diff --git a/src/test/datascience/dataviewer.functional.test.tsx b/src/test/datascience/dataviewer.functional.test.tsx index b2628ca60b62..5f86b4a4c1b6 100644 --- a/src/test/datascience/dataviewer.functional.test.tsx +++ b/src/test/datascience/dataviewer.functional.test.tsx @@ -4,7 +4,7 @@ // tslint:disable:max-func-body-length trailing-comma no-any no-multiline-string import '../../client/common/extensions'; -import { nbformat } from '@jupyterlab/coreutils'; +import { nbformat } from '@jupyterlab/services/node_modules/@jupyterlab/coreutils'; import * as assert from 'assert'; import { mount, ReactWrapper } from 'enzyme'; import { parse } from 'node-html-parser'; diff --git a/src/test/datascience/gather/gather.unit.test.ts b/src/test/datascience/gather/gather.unit.test.ts index a2a395a95594..81c23eb66fa2 100644 --- a/src/test/datascience/gather/gather.unit.test.ts +++ b/src/test/datascience/gather/gather.unit.test.ts @@ -131,7 +131,7 @@ suite('DataScience code gathering unit tests', () => { for (const c of codeCells) { await gatherExecution.postExecute(c, false); count += 1; - assert.equal(gatherExecution.executionSlicer._executionLog.length, count); + assert.equal(gatherExecution.executionSlicer.executionLog.length, count); } }); diff --git a/src/test/datascience/interactiveWindowCommandListener.unit.test.ts b/src/test/datascience/interactiveWindowCommandListener.unit.test.ts index 648708b8f7e1..5dd74c00271a 100644 --- a/src/test/datascience/interactiveWindowCommandListener.unit.test.ts +++ b/src/test/datascience/interactiveWindowCommandListener.unit.test.ts @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. 'use strict'; -import { nbformat } from '@jupyterlab/coreutils/lib/nbformat'; +import { nbformat } from '@jupyterlab/services/node_modules/@jupyterlab/coreutils/lib/nbformat'; import { assert } from 'chai'; import { anything, instance, mock, when } from 'ts-mockito'; import { Matcher } from 'ts-mockito/lib/matcher/type/Matcher'; diff --git a/src/test/datascience/jupyterVariables.unit.test.ts b/src/test/datascience/jupyterVariables.unit.test.ts index 5efba16b5190..e6f314d84a23 100644 --- a/src/test/datascience/jupyterVariables.unit.test.ts +++ b/src/test/datascience/jupyterVariables.unit.test.ts @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. 'use strict'; -import { nbformat } from '@jupyterlab/coreutils'; +import { nbformat } from '@jupyterlab/services/node_modules/@jupyterlab/coreutils'; import * as assert from 'assert'; import * as typemoq from 'typemoq'; diff --git a/src/test/datascience/mockJupyterManager.ts b/src/test/datascience/mockJupyterManager.ts index b42b97c0bbaf..6110b4a4cd41 100644 --- a/src/test/datascience/mockJupyterManager.ts +++ b/src/test/datascience/mockJupyterManager.ts @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. 'use strict'; -import { nbformat } from '@jupyterlab/coreutils'; +import { nbformat } from '@jupyterlab/services/node_modules/@jupyterlab/coreutils'; import { ChildProcess } from 'child_process'; import * as fs from 'fs-extra'; import * as os from 'os'; diff --git a/src/test/datascience/mockJupyterRequest.ts b/src/test/datascience/mockJupyterRequest.ts index 1620a069c695..6470cabc34d6 100644 --- a/src/test/datascience/mockJupyterRequest.ts +++ b/src/test/datascience/mockJupyterRequest.ts @@ -1,8 +1,8 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. 'use strict'; -import { nbformat } from '@jupyterlab/coreutils'; import { Kernel, KernelMessage } from '@jupyterlab/services'; +import { nbformat } from '@jupyterlab/services/node_modules/@jupyterlab/coreutils'; import { CancellationToken } from 'vscode-jsonrpc'; import { createDeferred, Deferred } from '../../client/common/utils/async'; diff --git a/src/test/datascience/notebook.functional.test.ts b/src/test/datascience/notebook.functional.test.ts index c6d31fbb9a82..845789fb3f64 100644 --- a/src/test/datascience/notebook.functional.test.ts +++ b/src/test/datascience/notebook.functional.test.ts @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. 'use strict'; -import { nbformat } from '@jupyterlab/coreutils'; +import { nbformat } from '@jupyterlab/services/node_modules/@jupyterlab/coreutils'; import { assert } from 'chai'; import { ChildProcess } from 'child_process'; import * as fs from 'fs-extra'; From d803bdbded2eedeb4e612869a4f77fde5de62db3 Mon Sep 17 00:00:00 2001 From: Jim Griesmer Date: Thu, 24 Oct 2019 23:50:51 -0700 Subject: [PATCH 02/42] Gather working (using 0.4.1 of python analysis) --- build/ci/postInstall.js | 2 +- package-lock.json | 6 +++--- package.json | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/build/ci/postInstall.js b/build/ci/postInstall.js index 8a26aab09d7a..3198674c4668 100644 --- a/build/ci/postInstall.js +++ b/build/ci/postInstall.js @@ -14,7 +14,7 @@ var constants_1 = require("../constants"); * The solution is to modify the type definition file after `npm install`. */ function fixJupyterLabDTSFiles() { - var relativePath = path.join('node_modules', '@jupyterlab', 'services', 'node_modules', 'coreutils', 'lb', 'settingregistry.d.ts'); + var relativePath = path.join('node_modules', '@jupyterlab', 'services', 'node_modules', '@jupyterlab', 'coreutils', 'lib', 'settingregistry.d.ts'); var filePath = path.join(constants_1.ExtensionRootDir, relativePath); if (!fs.existsSync(filePath)) { throw new Error("Type Definition file from JupyterLab not found '" + filePath + "' (pvsc post install script)"); diff --git a/package-lock.json b/package-lock.json index 86d010556993..888982bb18ad 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1271,9 +1271,9 @@ } }, "@msrvida/python-program-analysis": { - "version": "0.4.0", - "resolved": "https://registry.npmjs.org/@msrvida/python-program-analysis/-/python-program-analysis-0.4.0.tgz", - "integrity": "sha512-/bqastZynDio4DSmFQsRBiXlLl7FBrKnyUgJqZoRHw6Uzs+LF3AHMkt1NzdRn0qauBMX5G/wIzgDqL1pgSt4LA==" + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/@msrvida/python-program-analysis/-/python-program-analysis-0.4.1.tgz", + "integrity": "sha512-8jCtPTTxXyvN3udvz71inScvFvQil7Wh61newdrq79CjuV0W634GCDGuyGpI5G+kgX9PbBZPQJTc1+BMNMB5sQ==" }, "@nteract/markdown": { "version": "3.0.1", diff --git a/package.json b/package.json index 1ef3ca3f5494..2b45b57a1c3b 100644 --- a/package.json +++ b/package.json @@ -2654,7 +2654,7 @@ "scripts": { "package": "gulp clean && gulp prePublishBundle && vsce package -o ms-python-insiders.vsix", "compile": "tsc -watch -p ./", - "compile-webviews-watch": "npx npx -n --max_old_space_size=4096 webpack --config webpack.datascience-ui.config.js --watch", + "compile-webviews-watch": "npx npx --max_old_space_size=4096 webpack --config webpack.datascience-ui.config.js --watch", "dump-datascience-webpack-stats": "npx -n --max_old_space_size=4096 webpack --config webpack.datascience-ui.config.js --profile --json > tmp/ds-stats.json", "compile-webviews": "gulp compile-webviews", "compile-webviews-verbose": "npx webpack --config webpack.datascience-ui.config.js", From ed0ebd2a7cda7e8c5ea79f3a249314b8f1522207 Mon Sep 17 00:00:00 2001 From: Jim Griesmer Date: Fri, 25 Oct 2019 11:39:55 -0700 Subject: [PATCH 03/42] Prep for PR --- package.json | 2 +- src/client/datascience/gather/gather.ts | 16 ---------------- 2 files changed, 1 insertion(+), 17 deletions(-) diff --git a/package.json b/package.json index 2b45b57a1c3b..1ef3ca3f5494 100644 --- a/package.json +++ b/package.json @@ -2654,7 +2654,7 @@ "scripts": { "package": "gulp clean && gulp prePublishBundle && vsce package -o ms-python-insiders.vsix", "compile": "tsc -watch -p ./", - "compile-webviews-watch": "npx npx --max_old_space_size=4096 webpack --config webpack.datascience-ui.config.js --watch", + "compile-webviews-watch": "npx npx -n --max_old_space_size=4096 webpack --config webpack.datascience-ui.config.js --watch", "dump-datascience-webpack-stats": "npx -n --max_old_space_size=4096 webpack --config webpack.datascience-ui.config.js --profile --json > tmp/ds-stats.json", "compile-webviews": "gulp compile-webviews", "compile-webviews-verbose": "npx webpack --config webpack.datascience-ui.config.js", diff --git a/src/client/datascience/gather/gather.ts b/src/client/datascience/gather/gather.ts index f87955b2f643..5ace679e0536 100644 --- a/src/client/datascience/gather/gather.ts +++ b/src/client/datascience/gather/gather.ts @@ -89,22 +89,6 @@ export class GatherExecution implements IGatherExecution, INotebookExecutionLogg // Add a comment at the top of the file explaining what gather does const descriptor = '# This file contains the minimal amount of code required to produce the code cell you gathered.\n'; return descriptor.concat(program); - - // const slicedExecution: SlicedExecution = this._executionSlicer.sliceAllExecutions(gatherCell.executionEventId).pop(); - // const depCells = this._executionSlicer.getDependentCells(gatherCell.executionEventId); - - // const descriptor = '# This file contains the minimal amount of code required to produce the code cell you gathered.\n'; - // let script: string = `${descriptor}`; - // if (slicedExecution.cellSlices.length > 0) { - // slicedExecution.cellSlices.forEach((c: CellSlice) => { script = script.concat(c.cell.text); }); - // } - - // if (depCells.length > 0) { - // depCells.forEach((c) => { script = script.concat(c.text); }); - // } - // script = script.replace(/#%%/g, defaultCellMarker); - - // return script; } public get executionSlicer() { From dfc4319213e5519408e7de57b8fed7402cd82365 Mon Sep 17 00:00:00 2001 From: Jim Griesmer Date: Fri, 25 Oct 2019 17:58:07 -0700 Subject: [PATCH 04/42] Implement gather for native editor (first pass) --- package-lock.json | 74 ++++++++++++++++++- package.json | 3 +- src/client/datascience/cellFactory.ts | 54 ++++++++------ src/client/datascience/common.ts | 2 +- .../editor-integration/cellhashprovider.ts | 2 +- .../editor-integration/codeLensFactory.ts | 4 +- .../editor-integration/decorator.ts | 4 +- src/client/datascience/gather/gather.ts | 4 + .../datascience/gather/gatherListener.ts | 14 +++- .../interactive-common/interactiveBase.ts | 4 +- .../interactive-ipynb/nativeEditor.ts | 2 +- .../interactiveWindowCommandListener.ts | 4 +- .../datascience/jupyter/jupyterDebugger.ts | 2 +- .../datascience/jupyter/jupyterExporter.ts | 2 +- .../datascience/jupyter/jupyterImporter.ts | 2 +- .../datascience/jupyter/jupyterNotebook.ts | 2 +- .../datascience/jupyter/jupyterVariables.ts | 2 +- src/client/datascience/types.ts | 3 +- .../history-react/interactiveCell.tsx | 2 +- .../interactive-common/cellInput.tsx | 2 +- .../interactive-common/cellOutput.tsx | 2 +- .../interactive-common/mainState.ts | 2 +- .../native-editor/nativeCell.tsx | 11 ++- .../dataviewer.functional.test.tsx | 2 +- ...eractiveWindowCommandListener.unit.test.ts | 2 +- .../datascience/jupyterVariables.unit.test.ts | 2 +- src/test/datascience/mockJupyterManager.ts | 2 +- src/test/datascience/mockJupyterRequest.ts | 2 +- .../datascience/notebook.functional.test.ts | 2 +- 29 files changed, 157 insertions(+), 58 deletions(-) diff --git a/package-lock.json b/package-lock.json index 888982bb18ad..612a39c943a6 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1197,6 +1197,24 @@ "integrity": "sha512-EzRFg92bRSD1W/zeuNkeGwph0nkWf+pP2l/lYW4/5hav7RjKKBN5kV1Ix7Tvi0CMu3pC4Wi/U7rNisiJMR3ORg==", "dev": true }, + "@jupyterlab/coreutils": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/@jupyterlab/coreutils/-/coreutils-3.1.0.tgz", + "integrity": "sha512-ZqgzDUyanyvc86gtCrIbc1M6iniKHYmWNWHvWOcnq3KIP3wk3grchsTYPTfQDxcUS6F04baPGp/KohEU2ml40Q==", + "requires": { + "@phosphor/commands": "^1.6.3", + "@phosphor/coreutils": "^1.3.1", + "@phosphor/disposable": "^1.2.0", + "@phosphor/properties": "^1.1.3", + "@phosphor/signaling": "^1.2.3", + "ajv": "^6.5.5", + "json5": "^2.1.0", + "minimist": "~1.2.0", + "moment": "^2.24.0", + "path-posix": "~1.0.0", + "url-parse": "~1.4.3" + } + }, "@jupyterlab/observables": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/@jupyterlab/observables/-/observables-2.2.0.tgz", @@ -1435,6 +1453,43 @@ "@phosphor/algorithm": "^1.1.3" } }, + "@phosphor/commands": { + "version": "1.7.2", + "resolved": "https://registry.npmjs.org/@phosphor/commands/-/commands-1.7.2.tgz", + "integrity": "sha512-iSyBIWMHsus323BVEARBhuVZNnVel8USo+FIPaAxGcq+icTSSe6+NtSxVQSmZblGN6Qm4iw6I6VtiSx0e6YDgQ==", + "requires": { + "@phosphor/algorithm": "^1.2.0", + "@phosphor/coreutils": "^1.3.1", + "@phosphor/disposable": "^1.3.1", + "@phosphor/domutils": "^1.1.4", + "@phosphor/keyboard": "^1.1.3", + "@phosphor/signaling": "^1.3.1" + }, + "dependencies": { + "@phosphor/algorithm": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@phosphor/algorithm/-/algorithm-1.2.0.tgz", + "integrity": "sha512-C9+dnjXyU2QAkWCW6QVDGExk4hhwxzAKf5/FIuYlHAI9X5vFv99PYm0EREDxX1PbMuvfFBZhPNu0PvuSDQ7sFA==" + }, + "@phosphor/disposable": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/@phosphor/disposable/-/disposable-1.3.1.tgz", + "integrity": "sha512-0NGzoTXTOizWizK/brKKd5EjJhuuEH4903tLika7q6wl/u0tgneJlTh7R+MBVeih0iNxtuJAfBa3IEY6Qmj+Sw==", + "requires": { + "@phosphor/algorithm": "^1.2.0", + "@phosphor/signaling": "^1.3.1" + } + }, + "@phosphor/signaling": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/@phosphor/signaling/-/signaling-1.3.1.tgz", + "integrity": "sha512-Eq3wVCPQAhUd9+gUGaYygMr+ov7dhSGblSBXiDzpZlSIfa8OVD4P3cCvYXr/acDTNmZ/gHTcSFO8/n3rDkeXzg==", + "requires": { + "@phosphor/algorithm": "^1.2.0" + } + } + } + }, "@phosphor/coreutils": { "version": "1.3.1", "resolved": "https://registry.npmjs.org/@phosphor/coreutils/-/coreutils-1.3.1.tgz", @@ -1449,6 +1504,16 @@ "@phosphor/signaling": "^1.2.3" } }, + "@phosphor/domutils": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/@phosphor/domutils/-/domutils-1.1.4.tgz", + "integrity": "sha512-ivwq5TWjQpKcHKXO8PrMl+/cKqbgxPClPiCKc1gwbMd+6hnW5VLwNG0WBzJTxCzXK43HxX18oH+tOZ3E04wc3w==" + }, + "@phosphor/keyboard": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/@phosphor/keyboard/-/keyboard-1.1.3.tgz", + "integrity": "sha512-dzxC/PyHiD6mXaESRy6PZTd9JeK+diwG1pyngkyUf127IXOEzubTIbu52VSdpGBklszu33ws05BAGDa4oBE4mQ==" + }, "@phosphor/messaging": { "version": "1.2.3", "resolved": "https://registry.npmjs.org/@phosphor/messaging/-/messaging-1.2.3.tgz", @@ -1458,6 +1523,11 @@ "@phosphor/collections": "^1.1.3" } }, + "@phosphor/properties": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/@phosphor/properties/-/properties-1.1.3.tgz", + "integrity": "sha512-GiglqzU77s6+tFVt6zPq9uuyu/PLQPFcqZt914ZhJ4cN/7yNI/SLyMzpYZ56IRMXvzK9TUgbRna6URE3XAwFUg==" + }, "@phosphor/signaling": { "version": "1.2.3", "resolved": "https://registry.npmjs.org/@phosphor/signaling/-/signaling-1.2.3.tgz", @@ -11538,7 +11608,6 @@ "version": "2.1.0", "resolved": "https://registry.npmjs.org/json5/-/json5-2.1.0.tgz", "integrity": "sha512-8Mh9h6xViijj36g7Dxi+Y4S6hNGV96vcJZr/SrlHh1LR/pEn/8j/+qIBbs44YKl69Lrfctp4QD+AdWLTMqEZAQ==", - "dev": true, "requires": { "minimist": "^1.2.0" } @@ -12893,8 +12962,7 @@ "moment": { "version": "2.24.0", "resolved": "https://registry.npmjs.org/moment/-/moment-2.24.0.tgz", - "integrity": "sha512-bV7f+6l2QigeBBZSM/6yTNq4P2fNpSWj/0e7jQcy87A8e7o2nAfP/34/2ky5Vw4B9S446EtIhodAzkFCcR4dQg==", - "dev": true + "integrity": "sha512-bV7f+6l2QigeBBZSM/6yTNq4P2fNpSWj/0e7jQcy87A8e7o2nAfP/34/2ky5Vw4B9S446EtIhodAzkFCcR4dQg==" }, "monaco-editor": { "version": "0.16.2", diff --git a/package.json b/package.json index 1ef3ca3f5494..39529810a3c3 100644 --- a/package.json +++ b/package.json @@ -1695,7 +1695,7 @@ "python.dataScience.enableGather": { "type": "boolean", "default": false, - "description": "Enable code gathering for single cells in the interactive window (experimental).", + "description": "Enable code gather. For a gathered cell, it and all the code cells it depends on can be exported to a script or a notebook.", "scope": "resource" }, "python.dataScience.codeLenses": { @@ -2679,6 +2679,7 @@ "dependencies": { "@blueprintjs/select": "3.10.0", "@jupyterlab/services": "^3.2.1", + "@jupyterlab/coreutils": "^3.1.0", "@msrvida/python-program-analysis": "^0.4.1", "ansi-regex": "^4.1.0", "arch": "^2.1.0", diff --git a/src/client/datascience/cellFactory.ts b/src/client/datascience/cellFactory.ts index 6fcc20387ffb..dccdd5e728e2 100644 --- a/src/client/datascience/cellFactory.ts +++ b/src/client/datascience/cellFactory.ts @@ -107,45 +107,51 @@ export function hasCells(document: TextDocument, settings?: IDataScienceSettings return false; } -export function generateCellRanges(document: TextDocument, settings?: IDataScienceSettings): { range: Range; title: string; cell_type: string }[] { - // Implmentation of getCells here based on Don's Jupyter extension work +export function generateCellRangesFromString(source: string, settings?: IDataScienceSettings): { range: Range; title: string; cell_type: string }[] { const matcher = new CellMatcher(settings); - const cells : { range: Range; title: string; cell_type: string }[] = []; - for (let index = 0; index < document.lineCount; index += 1) { - const line = document.lineAt(index); - if (matcher.isCell(line.text)) { - if (cells.length > 0) { - const previousCell = cells[cells.length - 1]; - previousCell.range = new Range(previousCell.range.start, document.lineAt(index - 1).range.end); + const cells: { range: Range; title: string; cell_type: string }[] = []; + const lines: string[] = source.splitLines(); + + for (let index = 0; index < lines.length; index += 1) { + if (matcher.isCell(lines[index])) { + + // We have a cell, find the next cell + let j = index + 1; + while (j < lines.length && !matcher.isCell(lines[j])) { + j += 1; } - const results = matcher.exec(line.text); - if (results !== undefined) { + const result = matcher.exec(lines[index]); + if (result !== undefined) { cells.push({ - range: line.range, - title: results, - cell_type: matcher.getCellType(line.text) + range: new Range(index, 0, j - 1, lines[j - 1].length), + title: result, + cell_type: matcher.getCellType(lines[index]) }); } } } - if (cells.length >= 1) { - const line = document.lineAt(document.lineCount - 1); - const previousCell = cells[cells.length - 1]; - previousCell.range = new Range(previousCell.range.start, line.range.end); - } - return cells; } -export function generateCellsFromDocument(document: TextDocument, settings?: IDataScienceSettings): ICell[] { +export function generateCellsFromString(source: string, settings?: IDataScienceSettings): ICell[] { // Get our ranges. They'll determine our cells - const ranges = generateCellRanges(document, settings); + const ranges = generateCellRangesFromString(source, settings); + + const lines: string[] = source.splitLines(); // For each one, get its text and turn it into a cell return Array.prototype.concat(...ranges.map(r => { - const code = document.getText(r.range); - return generateCells(settings, code, document.fileName, r.range.start.line, false, uuid()); + const code = lines.slice(r.range.start.line, r.range.end.line).join(''); + return generateCells(settings, code, '', r.range.start.line, false, uuid()); })); } + +export function generateCellRangesFromDocument(document: TextDocument, settings?: IDataScienceSettings): { range: Range; title: string; cell_type: string }[] { + return generateCellRangesFromString(document.getText(), settings); +} + +export function generateCellsFromDocument(document: TextDocument, settings?: IDataScienceSettings): ICell[] { + return generateCellsFromString(document.getText(), settings); +} diff --git a/src/client/datascience/common.ts b/src/client/datascience/common.ts index 5f7092d8a3cd..fb2b611032a2 100644 --- a/src/client/datascience/common.ts +++ b/src/client/datascience/common.ts @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. 'use strict'; -import { nbformat } from '@jupyterlab/services/node_modules/@jupyterlab/coreutils/lib/nbformat'; +import { nbformat } from '@jupyterlab/coreutils/lib/nbformat'; import { noop } from '../../test/core'; diff --git a/src/client/datascience/editor-integration/cellhashprovider.ts b/src/client/datascience/editor-integration/cellhashprovider.ts index c088f441f7fa..ad0c7f210661 100644 --- a/src/client/datascience/editor-integration/cellhashprovider.ts +++ b/src/client/datascience/editor-integration/cellhashprovider.ts @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. 'use strict'; -import { nbformat } from '@jupyterlab/services/node_modules/@jupyterlab/coreutils'; +import { nbformat } from '@jupyterlab/coreutils'; import * as hashjs from 'hash.js'; import { inject, injectable, multiInject, optional } from 'inversify'; import { Event, EventEmitter, Position, Range, TextDocumentChangeEvent, TextDocumentContentChangeEvent } from 'vscode'; diff --git a/src/client/datascience/editor-integration/codeLensFactory.ts b/src/client/datascience/editor-integration/codeLensFactory.ts index 180711da5893..58750962bbba 100644 --- a/src/client/datascience/editor-integration/codeLensFactory.ts +++ b/src/client/datascience/editor-integration/codeLensFactory.ts @@ -9,7 +9,7 @@ import { IFileSystem } from '../../common/platform/types'; import { IConfigurationService } from '../../common/types'; import * as localize from '../../common/utils/localize'; import { noop } from '../../common/utils/misc'; -import { generateCellRanges } from '../cellFactory'; +import { generateCellRangesFromDocument } from '../cellFactory'; import { CodeLensCommands, Commands } from '../constants'; import { InteractiveWindowMessages } from '../interactive-common/interactiveWindowTypes'; import { ICell, ICellHashProvider, ICodeLensFactory, IFileHashes, IInteractiveWindowListener } from '../types'; @@ -63,7 +63,7 @@ export class CodeLensFactory implements ICodeLensFactory, IInteractiveWindowList } public createCodeLenses(document: TextDocument): CodeLens[] { - const ranges = generateCellRanges(document, this.configService.getSettings().datascience); + const ranges = generateCellRangesFromDocument(document, this.configService.getSettings().datascience); const commands = this.enumerateCommands(); const hashes = this.configService.getSettings().datascience.addGotoCodeLenses ? this.hashProvider.getHashes() : []; const codeLenses: CodeLens[] = []; diff --git a/src/client/datascience/editor-integration/decorator.ts b/src/client/datascience/editor-integration/decorator.ts index e8e62b5c1bd2..42c5cd3b7097 100644 --- a/src/client/datascience/editor-integration/decorator.ts +++ b/src/client/datascience/editor-integration/decorator.ts @@ -8,7 +8,7 @@ import { IExtensionSingleActivationService } from '../../activation/types'; import { IDocumentManager } from '../../common/application/types'; import { PYTHON_LANGUAGE } from '../../common/constants'; import { IConfigurationService, IDisposable, IDisposableRegistry } from '../../common/types'; -import { generateCellRanges } from '../cellFactory'; +import { generateCellRangesFromDocument } from '../cellFactory'; @injectable() export class Decorator implements IExtensionSingleActivationService, IDisposable { @@ -100,7 +100,7 @@ export class Decorator implements IExtensionSingleActivationService, IDisposable const settings = this.configuration.getSettings().datascience; if (settings.decorateCells && settings.enabled) { // Find all of the cells - const cells = generateCellRanges(editor.document, this.configuration.getSettings().datascience); + const cells = generateCellRangesFromDocument(editor.document, this.configuration.getSettings().datascience); // Find the range for our active cell. const currentRange = cells.map(c => c.range).filter(r => r.contains(editor.selection.anchor)); diff --git a/src/client/datascience/gather/gather.ts b/src/client/datascience/gather/gather.ts index 5ace679e0536..317ce423222e 100644 --- a/src/client/datascience/gather/gather.ts +++ b/src/client/datascience/gather/gather.ts @@ -69,6 +69,10 @@ export class GatherExecution implements IGatherExecution, INotebookExecutionLogg } } + public async resetLog(): Promise { + this._executionSlicer.reset(); + } + /** * For a given code cell, returns a string representing a program containing all the code it depends on. */ diff --git a/src/client/datascience/gather/gatherListener.ts b/src/client/datascience/gather/gatherListener.ts index c5639afe5b18..bb60b7346f23 100644 --- a/src/client/datascience/gather/gatherListener.ts +++ b/src/client/datascience/gather/gatherListener.ts @@ -4,6 +4,7 @@ import { IApplicationShell, IDocumentManager } from '../../common/application/ty import { PYTHON_LANGUAGE } from '../../common/constants'; import { IFileSystem } from '../../common/platform/types'; import { noop } from '../../common/utils/misc'; +import { generateCellsFromString } from '../cellFactory'; import { InteractiveWindowMessages } from '../interactive-common/interactiveWindowTypes'; import { ICell, IGatherExecution, IInteractiveWindowListener } from '../types'; @@ -16,8 +17,7 @@ export class GatherListener implements IInteractiveWindowListener { @inject(IDocumentManager) private documentManager: IDocumentManager, @inject(IApplicationShell) private applicationShell: IApplicationShell, @inject(IGatherExecution) private gatherExecution: IGatherExecution, - @inject(IFileSystem) private fileSystem: IFileSystem - ) { } + @inject(IFileSystem) private fileSystem: IFileSystem) { } public dispose() { noop(); @@ -38,6 +38,10 @@ export class GatherListener implements IInteractiveWindowListener { } break; + case InteractiveWindowMessages.RestartKernel: + this.gatherExecution.resetLog(); + break; + default: break; } @@ -67,6 +71,12 @@ export class GatherListener implements IInteractiveWindowListener { viewColumn = ViewColumn.Beside; } + // Create new notebook with the returned program and open it. + const cells: ICell[] = generateCellsFromString(slicedProgram); + if (cells.length === 0) { + return; + } + // Create a new open editor with the returned program in the right panel const doc = await this.documentManager.openTextDocument({ content: slicedProgram, diff --git a/src/client/datascience/interactive-common/interactiveBase.ts b/src/client/datascience/interactive-common/interactiveBase.ts index 126fe652feac..329d062c3330 100644 --- a/src/client/datascience/interactive-common/interactiveBase.ts +++ b/src/client/datascience/interactive-common/interactiveBase.ts @@ -29,7 +29,7 @@ import * as localize from '../../common/utils/localize'; import { StopWatch } from '../../common/utils/stopWatch'; import { IInterpreterService, PythonInterpreter } from '../../interpreter/contracts'; import { captureTelemetry, sendTelemetryEvent } from '../../telemetry'; -import { generateCellRanges } from '../cellFactory'; +import { generateCellRangesFromDocument } from '../cellFactory'; import { CellMatcher } from '../cellMatcher'; import { Identifiers, Telemetry } from '../constants'; import { ColumnWarningSize } from '../data-viewing/types'; @@ -1013,7 +1013,7 @@ export abstract class InteractiveBase extends WebViewHost 0; const line = editor.selection.start.line; const revealLine = line + 1; diff --git a/src/client/datascience/interactive-ipynb/nativeEditor.ts b/src/client/datascience/interactive-ipynb/nativeEditor.ts index bf37fd64eacc..8342e6e70de1 100644 --- a/src/client/datascience/interactive-ipynb/nativeEditor.ts +++ b/src/client/datascience/interactive-ipynb/nativeEditor.ts @@ -3,7 +3,7 @@ 'use strict'; import '../../common/extensions'; -import { nbformat } from '@jupyterlab/services/node_modules/@jupyterlab/coreutils/lib/nbformat'; +import { nbformat } from '@jupyterlab/coreutils/lib/nbformat'; import * as detectIndent from 'detect-indent'; import { inject, injectable, multiInject, named } from 'inversify'; import * as path from 'path'; diff --git a/src/client/datascience/interactive-window/interactiveWindowCommandListener.ts b/src/client/datascience/interactive-window/interactiveWindowCommandListener.ts index 4933f6dde117..e30270517039 100644 --- a/src/client/datascience/interactive-window/interactiveWindowCommandListener.ts +++ b/src/client/datascience/interactive-window/interactiveWindowCommandListener.ts @@ -16,7 +16,7 @@ import { IConfigurationService, IDisposableRegistry, ILogger } from '../../commo import * as localize from '../../common/utils/localize'; import { captureTelemetry } from '../../telemetry'; import { CommandSource } from '../../testing/common/constants'; -import { generateCellRanges, generateCellsFromDocument } from '../cellFactory'; +import { generateCellRangesFromDocument, generateCellsFromDocument } from '../cellFactory'; import { Commands, Identifiers, Telemetry } from '../constants'; import { JupyterInstallError } from '../jupyter/jupyterInstallError'; import { @@ -194,7 +194,7 @@ export class InteractiveWindowCommandListener implements IDataScienceCommandList // If the current file is the active editor, then generate cells from the document. const activeEditor = this.documentManager.activeTextEditor; if (activeEditor && activeEditor.document && this.fileSystem.arePathsSame(activeEditor.document.fileName, file)) { - const ranges = generateCellRanges(activeEditor.document); + const ranges = generateCellRangesFromDocument(activeEditor.document); if (ranges.length > 0) { // Ask user for path const output = await this.showExportDialog(); diff --git a/src/client/datascience/jupyter/jupyterDebugger.ts b/src/client/datascience/jupyter/jupyterDebugger.ts index 31001309db0d..56afb727606e 100644 --- a/src/client/datascience/jupyter/jupyterDebugger.ts +++ b/src/client/datascience/jupyter/jupyterDebugger.ts @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. 'use strict'; -import { nbformat } from '@jupyterlab/services/node_modules/@jupyterlab/coreutils'; +import { nbformat } from '@jupyterlab/coreutils'; import { inject, injectable } from 'inversify'; import * as uuid from 'uuid/v4'; import { DebugConfiguration } from 'vscode'; diff --git a/src/client/datascience/jupyter/jupyterExporter.ts b/src/client/datascience/jupyter/jupyterExporter.ts index 0116ad43898f..89fed8dfbbf5 100644 --- a/src/client/datascience/jupyter/jupyterExporter.ts +++ b/src/client/datascience/jupyter/jupyterExporter.ts @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. 'use strict'; -import { nbformat } from '@jupyterlab/services/node_modules/@jupyterlab/coreutils'; +import { nbformat } from '@jupyterlab/coreutils'; import { inject, injectable } from 'inversify'; import * as os from 'os'; import * as path from 'path'; diff --git a/src/client/datascience/jupyter/jupyterImporter.ts b/src/client/datascience/jupyter/jupyterImporter.ts index b7e5e9a3de5c..b0b644207634 100644 --- a/src/client/datascience/jupyter/jupyterImporter.ts +++ b/src/client/datascience/jupyter/jupyterImporter.ts @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. 'use strict'; -import { nbformat } from '@jupyterlab/services/node_modules/@jupyterlab/coreutils'; +import { nbformat } from '@jupyterlab/coreutils'; import * as fs from 'fs-extra'; import { inject, injectable } from 'inversify'; import * as os from 'os'; diff --git a/src/client/datascience/jupyter/jupyterNotebook.ts b/src/client/datascience/jupyter/jupyterNotebook.ts index f8597bda28fc..12dbeac05e87 100644 --- a/src/client/datascience/jupyter/jupyterNotebook.ts +++ b/src/client/datascience/jupyter/jupyterNotebook.ts @@ -3,8 +3,8 @@ 'use strict'; import '../../common/extensions'; +import { nbformat } from '@jupyterlab/coreutils'; import { Kernel, KernelMessage } from '@jupyterlab/services'; -import { nbformat } from '@jupyterlab/services/node_modules/@jupyterlab/coreutils'; import * as fs from 'fs-extra'; import { Observable } from 'rxjs/Observable'; import { Subscriber } from 'rxjs/Subscriber'; diff --git a/src/client/datascience/jupyter/jupyterVariables.ts b/src/client/datascience/jupyter/jupyterVariables.ts index 2923b7736bfc..f452ec922877 100644 --- a/src/client/datascience/jupyter/jupyterVariables.ts +++ b/src/client/datascience/jupyter/jupyterVariables.ts @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. 'use strict'; -import { nbformat } from '@jupyterlab/services/node_modules/@jupyterlab/coreutils'; +import { nbformat } from '@jupyterlab/coreutils'; import { JSONObject } from '@phosphor/coreutils'; import { inject, injectable } from 'inversify'; import * as path from 'path'; diff --git a/src/client/datascience/types.ts b/src/client/datascience/types.ts index 2027a8fb3e68..1e83d0cc5a3f 100644 --- a/src/client/datascience/types.ts +++ b/src/client/datascience/types.ts @@ -1,8 +1,8 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. 'use strict'; +import { nbformat } from '@jupyterlab/coreutils'; import { Kernel, KernelMessage } from '@jupyterlab/services/lib/kernel'; -import { nbformat } from '@jupyterlab/services/node_modules/@jupyterlab/coreutils'; import { JSONObject } from '@phosphor/coreutils'; import { Observable } from 'rxjs/Observable'; import { @@ -119,6 +119,7 @@ export const IGatherExecution = Symbol('IGatherExecution'); export interface IGatherExecution { enabled: boolean; gatherCode(vscCell: ICell): string; + resetLog(): void; } export const IJupyterExecution = Symbol('IJupyterExecution'); diff --git a/src/datascience-ui/history-react/interactiveCell.tsx b/src/datascience-ui/history-react/interactiveCell.tsx index 473d9f55ca9e..a437b975db2d 100644 --- a/src/datascience-ui/history-react/interactiveCell.tsx +++ b/src/datascience-ui/history-react/interactiveCell.tsx @@ -3,7 +3,7 @@ 'use strict'; import '../../client/common/extensions'; -import { nbformat } from '@jupyterlab/services/node_modules/@jupyterlab/coreutils'; +import { nbformat } from '@jupyterlab/coreutils'; import * as fastDeepEqual from 'fast-deep-equal'; import * as monacoEditor from 'monaco-editor/esm/vs/editor/editor.api'; import * as React from 'react'; diff --git a/src/datascience-ui/interactive-common/cellInput.tsx b/src/datascience-ui/interactive-common/cellInput.tsx index 6e65d1ada50d..41ec875fcd1e 100644 --- a/src/datascience-ui/interactive-common/cellInput.tsx +++ b/src/datascience-ui/interactive-common/cellInput.tsx @@ -3,7 +3,7 @@ 'use strict'; import '../../client/common/extensions'; -import { nbformat } from '@jupyterlab/services/node_modules/@jupyterlab/coreutils'; +import { nbformat } from '@jupyterlab/coreutils'; import * as monacoEditor from 'monaco-editor/esm/vs/editor/editor.api'; import * as React from 'react'; diff --git a/src/datascience-ui/interactive-common/cellOutput.tsx b/src/datascience-ui/interactive-common/cellOutput.tsx index 4c406c254d35..c3e978f0a670 100644 --- a/src/datascience-ui/interactive-common/cellOutput.tsx +++ b/src/datascience-ui/interactive-common/cellOutput.tsx @@ -6,7 +6,7 @@ import '../../client/common/extensions'; // tslint:disable-next-line: no-var-requires no-require-imports const ansiToHtml = require('ansi-to-html'); -import { nbformat } from '@jupyterlab/services/node_modules/@jupyterlab/coreutils'; +import { nbformat } from '@jupyterlab/coreutils'; import { JSONObject } from '@phosphor/coreutils'; import ansiRegex from 'ansi-regex'; // tslint:disable-next-line: no-require-imports diff --git a/src/datascience-ui/interactive-common/mainState.ts b/src/datascience-ui/interactive-common/mainState.ts index 29d59fb4c026..5f1767669169 100644 --- a/src/datascience-ui/interactive-common/mainState.ts +++ b/src/datascience-ui/interactive-common/mainState.ts @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. 'use strict'; -import { nbformat } from '@jupyterlab/services/node_modules/@jupyterlab/coreutils'; +import { nbformat } from '@jupyterlab/coreutils'; // tslint:disable-next-line: no-require-imports import cloneDeep = require('lodash/cloneDeep'); import * as monacoEditor from 'monaco-editor/esm/vs/editor/editor.api'; diff --git a/src/datascience-ui/native-editor/nativeCell.tsx b/src/datascience-ui/native-editor/nativeCell.tsx index b767d12bca56..38da579810a5 100644 --- a/src/datascience-ui/native-editor/nativeCell.tsx +++ b/src/datascience-ui/native-editor/nativeCell.tsx @@ -3,7 +3,7 @@ 'use strict'; import '../../client/common/extensions'; -import { nbformat } from '@jupyterlab/services/node_modules/@jupyterlab/coreutils'; +import { nbformat } from '@jupyterlab/coreutils'; import * as fastDeepEqual from 'fast-deep-equal'; import * as React from 'react'; @@ -557,6 +557,9 @@ export class NativeCell extends React.Component { private renderMiddleToolbar = () => { const cellId = this.props.cellVM.cell.id; + const gatherCell = () => { + this.props.stateController.gatherCell(this.props.cellVM); + }; const deleteCell = () => { this.props.stateController.possiblyDeleteCell(cellId); this.props.stateController.sendCommand(NativeCommandType.DeleteCell, 'mouse'); @@ -569,6 +572,9 @@ export class NativeCell extends React.Component { this.props.stateController.runBelow(cellId); this.props.stateController.sendCommand(NativeCommandType.RunBelow, 'mouse'); }; + const gatherDisabled = () => { + return this.props.cellVM.cell.data.execution_count === null || this.props.cellVM.cell.data.cell_type !== 'code' || getSettings().enableGather === false; + }; const canRunAbove = this.props.stateController.canRunAbove(cellId); const canRunBelow = this.props.cellVM.cell.state === CellState.finished || this.props.cellVM.cell.state === CellState.error; const switchTooltip = this.props.cellVM.cell.data.cell_type === 'code' ? getLocString('DataScience.switchToMarkdown', 'Change to markdown') : @@ -615,6 +621,9 @@ export class NativeCell extends React.Component { + ); } diff --git a/src/test/datascience/dataviewer.functional.test.tsx b/src/test/datascience/dataviewer.functional.test.tsx index 5f86b4a4c1b6..b2628ca60b62 100644 --- a/src/test/datascience/dataviewer.functional.test.tsx +++ b/src/test/datascience/dataviewer.functional.test.tsx @@ -4,7 +4,7 @@ // tslint:disable:max-func-body-length trailing-comma no-any no-multiline-string import '../../client/common/extensions'; -import { nbformat } from '@jupyterlab/services/node_modules/@jupyterlab/coreutils'; +import { nbformat } from '@jupyterlab/coreutils'; import * as assert from 'assert'; import { mount, ReactWrapper } from 'enzyme'; import { parse } from 'node-html-parser'; diff --git a/src/test/datascience/interactiveWindowCommandListener.unit.test.ts b/src/test/datascience/interactiveWindowCommandListener.unit.test.ts index 5dd74c00271a..648708b8f7e1 100644 --- a/src/test/datascience/interactiveWindowCommandListener.unit.test.ts +++ b/src/test/datascience/interactiveWindowCommandListener.unit.test.ts @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. 'use strict'; -import { nbformat } from '@jupyterlab/services/node_modules/@jupyterlab/coreutils/lib/nbformat'; +import { nbformat } from '@jupyterlab/coreutils/lib/nbformat'; import { assert } from 'chai'; import { anything, instance, mock, when } from 'ts-mockito'; import { Matcher } from 'ts-mockito/lib/matcher/type/Matcher'; diff --git a/src/test/datascience/jupyterVariables.unit.test.ts b/src/test/datascience/jupyterVariables.unit.test.ts index e6f314d84a23..5efba16b5190 100644 --- a/src/test/datascience/jupyterVariables.unit.test.ts +++ b/src/test/datascience/jupyterVariables.unit.test.ts @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. 'use strict'; -import { nbformat } from '@jupyterlab/services/node_modules/@jupyterlab/coreutils'; +import { nbformat } from '@jupyterlab/coreutils'; import * as assert from 'assert'; import * as typemoq from 'typemoq'; diff --git a/src/test/datascience/mockJupyterManager.ts b/src/test/datascience/mockJupyterManager.ts index 6110b4a4cd41..b42b97c0bbaf 100644 --- a/src/test/datascience/mockJupyterManager.ts +++ b/src/test/datascience/mockJupyterManager.ts @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. 'use strict'; -import { nbformat } from '@jupyterlab/services/node_modules/@jupyterlab/coreutils'; +import { nbformat } from '@jupyterlab/coreutils'; import { ChildProcess } from 'child_process'; import * as fs from 'fs-extra'; import * as os from 'os'; diff --git a/src/test/datascience/mockJupyterRequest.ts b/src/test/datascience/mockJupyterRequest.ts index 6470cabc34d6..1620a069c695 100644 --- a/src/test/datascience/mockJupyterRequest.ts +++ b/src/test/datascience/mockJupyterRequest.ts @@ -1,8 +1,8 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. 'use strict'; +import { nbformat } from '@jupyterlab/coreutils'; import { Kernel, KernelMessage } from '@jupyterlab/services'; -import { nbformat } from '@jupyterlab/services/node_modules/@jupyterlab/coreutils'; import { CancellationToken } from 'vscode-jsonrpc'; import { createDeferred, Deferred } from '../../client/common/utils/async'; diff --git a/src/test/datascience/notebook.functional.test.ts b/src/test/datascience/notebook.functional.test.ts index 845789fb3f64..c6d31fbb9a82 100644 --- a/src/test/datascience/notebook.functional.test.ts +++ b/src/test/datascience/notebook.functional.test.ts @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. 'use strict'; -import { nbformat } from '@jupyterlab/services/node_modules/@jupyterlab/coreutils'; +import { nbformat } from '@jupyterlab/coreutils'; import { assert } from 'chai'; import { ChildProcess } from 'child_process'; import * as fs from 'fs-extra'; From 6b4e1eaff6734f5ee35efb73140711f190db25e0 Mon Sep 17 00:00:00 2001 From: Jim Griesmer Date: Sat, 26 Oct 2019 12:38:41 -0700 Subject: [PATCH 05/42] Fix cell rangification --- src/client/datascience/cellFactory.ts | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/src/client/datascience/cellFactory.ts b/src/client/datascience/cellFactory.ts index dccdd5e728e2..efced2ae66ab 100644 --- a/src/client/datascience/cellFactory.ts +++ b/src/client/datascience/cellFactory.ts @@ -4,7 +4,7 @@ import '../common/extensions'; import * as uuid from 'uuid/v4'; -import { Range, TextDocument } from 'vscode'; +import { Range, TextDocument, Position } from 'vscode'; import { noop } from '../../test/core'; import { IDataScienceSettings } from '../common/types'; @@ -110,11 +110,17 @@ export function hasCells(document: TextDocument, settings?: IDataScienceSettings export function generateCellRangesFromString(source: string, settings?: IDataScienceSettings): { range: Range; title: string; cell_type: string }[] { const matcher = new CellMatcher(settings); const cells: { range: Range; title: string; cell_type: string }[] = []; - const lines: string[] = source.splitLines(); + const lines: string[] = source.splitLines({ trim: true, removeEmptyEntries: false }); for (let index = 0; index < lines.length; index += 1) { if (matcher.isCell(lines[index])) { + // // If we have at least one cell... + // if (cells.length > 0) { + // const previousCell = cells[cells.length - 1]; + // previousCell.range = new Range(previousCell.range.start, new Position(index - 1, lines[index - 1].length)); + // } + // We have a cell, find the next cell let j = index + 1; while (j < lines.length && !matcher.isCell(lines[j])) { @@ -132,6 +138,12 @@ export function generateCellRangesFromString(source: string, settings?: IDataSci } } + // if (cells.length >= 1) { + // const previousCell = cells[cells.length - 1]; + // previousCell.range = new Range(previousCell.range.start, new Position(lines.length - 1, lines[lines.length - 1].length)); + // } + + return cells; } From b827c1a9b35ee64d2feb211d7f01eddeef9bd932 Mon Sep 17 00:00:00 2001 From: Jim Griesmer Date: Sat, 26 Oct 2019 12:39:16 -0700 Subject: [PATCH 06/42] Cleanup --- src/client/datascience/cellFactory.ts | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/src/client/datascience/cellFactory.ts b/src/client/datascience/cellFactory.ts index efced2ae66ab..537922eed8b0 100644 --- a/src/client/datascience/cellFactory.ts +++ b/src/client/datascience/cellFactory.ts @@ -115,12 +115,6 @@ export function generateCellRangesFromString(source: string, settings?: IDataSci for (let index = 0; index < lines.length; index += 1) { if (matcher.isCell(lines[index])) { - // // If we have at least one cell... - // if (cells.length > 0) { - // const previousCell = cells[cells.length - 1]; - // previousCell.range = new Range(previousCell.range.start, new Position(index - 1, lines[index - 1].length)); - // } - // We have a cell, find the next cell let j = index + 1; while (j < lines.length && !matcher.isCell(lines[j])) { @@ -138,12 +132,6 @@ export function generateCellRangesFromString(source: string, settings?: IDataSci } } - // if (cells.length >= 1) { - // const previousCell = cells[cells.length - 1]; - // previousCell.range = new Range(previousCell.range.start, new Position(lines.length - 1, lines[lines.length - 1].length)); - // } - - return cells; } From 30f1fccb564d0c23e59b72d8b9f98eb7775e29fe Mon Sep 17 00:00:00 2001 From: Jim Griesmer Date: Sat, 26 Oct 2019 17:17:08 -0700 Subject: [PATCH 07/42] Fix error --- src/client/datascience/cellFactory.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/client/datascience/cellFactory.ts b/src/client/datascience/cellFactory.ts index 537922eed8b0..a62fe2ef2fa9 100644 --- a/src/client/datascience/cellFactory.ts +++ b/src/client/datascience/cellFactory.ts @@ -4,7 +4,7 @@ import '../common/extensions'; import * as uuid from 'uuid/v4'; -import { Range, TextDocument, Position } from 'vscode'; +import { Range, TextDocument } from 'vscode'; import { noop } from '../../test/core'; import { IDataScienceSettings } from '../common/types'; From 338361f184a7211d69fc942dfc524984f310bc21 Mon Sep 17 00:00:00 2001 From: Jim Griesmer Date: Sun, 27 Oct 2019 17:31:39 -0700 Subject: [PATCH 08/42] Support knowng cell has been exec'd --- src/client/datascience/cellFactory.ts | 4 +- .../datascience/gather/gatherListener.ts | 10 +- .../interactive-common/interactiveBase.ts | 2 + .../interactive-ipynb/nativeEditor.ts | 24 ++++- .../interactive-ipynb/nativeEditorProvider.ts | 98 +++++++++---------- .../datascience/jupyter/jupyterExporter.ts | 6 +- .../datascience/jupyter/jupyterImporter.ts | 1 + .../datascience/jupyter/jupyterNotebook.ts | 3 +- src/client/datascience/types.ts | 2 + .../interactive-common/mainState.ts | 4 +- .../native-editor/nativeCell.tsx | 5 +- .../cellhashprovider.unit.test.ts | 3 +- .../datascience/gather/gather.unit.test.ts | 5 + .../datascience/jupyterVariables.unit.test.ts | 3 +- .../datascience/testNativeEditorProvider.ts | 4 + 15 files changed, 111 insertions(+), 63 deletions(-) diff --git a/src/client/datascience/cellFactory.ts b/src/client/datascience/cellFactory.ts index a62fe2ef2fa9..662d888666d3 100644 --- a/src/client/datascience/cellFactory.ts +++ b/src/client/datascience/cellFactory.ts @@ -43,7 +43,8 @@ function generateCodeCell(code: string[], file: string, line: number, id: string id: id, file: file, line: line, - state: CellState.init + state: CellState.init, + executedInCurrentKernel: false }; } @@ -54,6 +55,7 @@ function generateMarkdownCell(code: string[], file: string, line: number, id: st file: file, line: line, state: CellState.finished, + executedInCurrentKernel: false, data: { cell_type: 'markdown', source: generateMarkdownFromCodeLines(code), diff --git a/src/client/datascience/gather/gatherListener.ts b/src/client/datascience/gather/gatherListener.ts index bb60b7346f23..6849f9138180 100644 --- a/src/client/datascience/gather/gatherListener.ts +++ b/src/client/datascience/gather/gatherListener.ts @@ -6,7 +6,7 @@ import { IFileSystem } from '../../common/platform/types'; import { noop } from '../../common/utils/misc'; import { generateCellsFromString } from '../cellFactory'; import { InteractiveWindowMessages } from '../interactive-common/interactiveWindowTypes'; -import { ICell, IGatherExecution, IInteractiveWindowListener } from '../types'; +import { ICell, IGatherExecution, IInteractiveWindowListener, INotebookEditorProvider, INotebookExporter } from '../types'; @injectable() export class GatherListener implements IInteractiveWindowListener { @@ -14,6 +14,8 @@ export class GatherListener implements IInteractiveWindowListener { private postEmitter: EventEmitter<{ message: string; payload: any }> = new EventEmitter<{ message: string; payload: any }>(); constructor( + @inject(INotebookExporter) private jupyterExporter: INotebookExporter, + @inject(INotebookEditorProvider) private ipynbProvider: INotebookEditorProvider, @inject(IDocumentManager) private documentManager: IDocumentManager, @inject(IApplicationShell) private applicationShell: IApplicationShell, @inject(IGatherExecution) private gatherExecution: IGatherExecution, @@ -77,6 +79,12 @@ export class GatherListener implements IInteractiveWindowListener { return; } + const notebook = await this.jupyterExporter.translateToNotebook(cells); + const contents = JSON.stringify(notebook); + + const newuri = await this.ipynbProvider.getNextNewNotebookUri(); + await (await this.ipynbProvider.open(newuri, contents)).show(); + // Create a new open editor with the returned program in the right panel const doc = await this.documentManager.openTextDocument({ content: slicedProgram, diff --git a/src/client/datascience/interactive-common/interactiveBase.ts b/src/client/datascience/interactive-common/interactiveBase.ts index 329d062c3330..4e80799be373 100644 --- a/src/client/datascience/interactive-common/interactiveBase.ts +++ b/src/client/datascience/interactive-common/interactiveBase.ts @@ -561,6 +561,7 @@ export abstract class InteractiveBase extends WebViewHost { - const data = this.globalStorage.get<{contents?: string; lastModifiedTimeMs?: number}>(this.getStorageKey()); + const data = this.globalStorage.get<{ contents?: string; lastModifiedTimeMs?: number }>(this.getStorageKey()); // Check whether the file has been modified since the last time the contents were saved. - if (data && data.lastModifiedTimeMs && !this.isUntitled && this.file.scheme === 'file'){ + if (data && data.lastModifiedTimeMs && !this.isUntitled && this.file.scheme === 'file') { const stat = await this.fileSystem.stat(this.file.fsPath); - if (stat.mtime > data.lastModifiedTimeMs){ + if (stat.mtime > data.lastModifiedTimeMs) { return; } } @@ -585,7 +592,7 @@ export class NativeEditor extends InteractiveBase implements INotebookEditor { const key = this.getStorageKey(); // Keep track of the time when this data was saved. // This way when we retrieve the data we can compare it against last modified date of the file. - await this.globalStorage.update(key, {contents, lastModifiedTimeMs: Date.now()}); + await this.globalStorage.update(key, { contents, lastModifiedTimeMs: Date.now() }); } private async close(): Promise { @@ -848,4 +855,11 @@ export class NativeEditor extends InteractiveBase implements INotebookEditor { sendTelemetryEvent(Telemetry.NotebookOpenTime, this.startupTimer.elapsedTime); } } + + private onRestartKernel() { + if (this.cells.length > 0) { + this.cells.forEach((c) => { c.executedInCurrentKernel = false; }); + } + this.sendCellsToWebView(this.cells); + } } diff --git a/src/client/datascience/interactive-ipynb/nativeEditorProvider.ts b/src/client/datascience/interactive-ipynb/nativeEditorProvider.ts index 7031c91578c0..06319e9078b9 100644 --- a/src/client/datascience/interactive-ipynb/nativeEditorProvider.ts +++ b/src/client/datascience/interactive-ipynb/nativeEditorProvider.ts @@ -17,6 +17,17 @@ import { IDataScienceErrorHandler, INotebookEditor, INotebookEditorProvider, INo @injectable() export class NativeEditorProvider implements INotebookEditorProvider, IAsyncDisposable { + + public get activeEditor(): INotebookEditor | undefined { + const active = [...this.activeEditors.entries()].find(e => e[1].active); + if (active) { + return active[1]; + } + } + + public get editors(): INotebookEditor[] { + return [...this.activeEditors.values()]; + } private activeEditors: Map = new Map(); private executedEditors: Set = new Set(); private notebookCount: number = 0; @@ -72,17 +83,6 @@ export class NativeEditorProvider implements INotebookEditorProvider, IAsyncDisp sendTelemetryEvent(Telemetry.NotebookWorkspaceCount, this.notebookCount); } - public get activeEditor(): INotebookEditor | undefined { - const active = [...this.activeEditors.entries()].find(e => e[1].active); - if (active) { - return active[1]; - } - } - - public get editors(): INotebookEditor[] { - return [...this.activeEditors.values()]; - } - public async open(file: Uri, contents: string): Promise { // See if this file is open or not already let editor = this.activeEditors.get(file.fsPath); @@ -129,15 +129,38 @@ export class NativeEditorProvider implements INotebookEditorProvider, IAsyncDisp purpose: Identifiers.HistoryPurpose // Share the same one as the interactive window. Just need a new session }; } + + public async getNextNewNotebookUri(): Promise { + // Start in the root and look for files starting with untitled + let number = 1; + const dir = this.workspace.rootPath; + if (dir) { + const existing = await this.fileSystem.search(`${dir}/${localize.DataScience.untitledNotebookFileName()}-*.ipynb`); + + // Sort by number + existing.sort(); + + // Add one onto the end of the last one + if (existing.length > 0) { + const match = /(\w+)-(\d+)\.ipynb/.exec(path.basename(existing[existing.length - 1])); + if (match && match.length > 1) { + number = parseInt(match[2], 10); + } + } + return Uri.file(path.join(dir, `${localize.DataScience.untitledNotebookFileName()}-${number}`)); + } + + return Uri.file(`${localize.DataScience.untitledNotebookFileName()}-${number}`); + } /** * Open ipynb files when user opens an ipynb file. * * @private * @memberof NativeEditorProvider */ - private onDidChangeActiveTextEditorHandler(editor?: TextEditor){ + private onDidChangeActiveTextEditorHandler(editor?: TextEditor) { // I we're a source control diff view, then ignore this editor. - if (!editor || this.isEditorPartOfDiffView(editor)){ + if (!editor || this.isEditorPartOfDiffView(editor)) { return; } this.openNotebookAndCloseEditor(editor.document, true).ignoreErrors(); @@ -165,40 +188,17 @@ export class NativeEditorProvider implements INotebookEditorProvider, IAsyncDisp this.openedNotebookCount += 1; } - private async getNextNewNotebookUri(): Promise { - // Start in the root and look for files starting with untitled - let number = 1; - const dir = this.workspace.rootPath; - if (dir) { - const existing = await this.fileSystem.search(`${dir}/${localize.DataScience.untitledNotebookFileName()}-*.ipynb`); - - // Sort by number - existing.sort(); - - // Add one onto the end of the last one - if (existing.length > 0) { - const match = /(\w+)-(\d+)\.ipynb/.exec(path.basename(existing[existing.length - 1])); - if (match && match.length > 1) { - number = parseInt(match[2], 10); - } - } - return Uri.file(path.join(dir, `${localize.DataScience.untitledNotebookFileName()}-${number}`)); - } - - return Uri.file(`${localize.DataScience.untitledNotebookFileName()}-${number}`); - } - private openNotebookAndCloseEditor = async (document: TextDocument, closeDocumentBeforeOpeningNotebook: boolean) => { // See if this is an ipynb file if (this.isNotebook(document) && this.configuration.getSettings().datascience.useNotebookEditor && !this.activeEditors.has(document.uri.fsPath)) { try { - const contents = document. getText(); + const contents = document.getText(); const uri = document.uri; const closeActiveEditorCommand = 'workbench.action.closeActiveEditor'; if (closeDocumentBeforeOpeningNotebook) { - if (!this.documentManager.activeTextEditor || this.documentManager.activeTextEditor.document !== document){ + if (!this.documentManager.activeTextEditor || this.documentManager.activeTextEditor.document !== document) { await this.documentManager.showTextDocument(document); } await this.cmdManager.executeCommand(closeActiveEditorCommand); @@ -207,7 +207,7 @@ export class NativeEditorProvider implements INotebookEditorProvider, IAsyncDisp // Open our own editor. await this.open(uri, contents); - if (!closeDocumentBeforeOpeningNotebook){ + if (!closeDocumentBeforeOpeningNotebook) { // Then switch back to the ipynb and close it. // If we don't do it in this order, the close will switch to the wrong item await this.documentManager.showTextDocument(document); @@ -226,14 +226,14 @@ export class NativeEditorProvider implements INotebookEditorProvider, IAsyncDisp * @param {TextEditor} editor * @memberof NativeEditorProvider */ - private isEditorPartOfDiffView(editor?: TextEditor){ - if (!editor){ + private isEditorPartOfDiffView(editor?: TextEditor) { + if (!editor) { return false; } // There's no easy way to determine if the user is openeing a diff view. // One simple way is to check if there are 2 editor opened, and if both editors point to the same file // One file with the `file` scheme and the other with the `git` scheme. - if (this.documentManager.visibleTextEditors.length <= 1){ + if (this.documentManager.visibleTextEditors.length <= 1) { return false; } @@ -242,18 +242,18 @@ export class NativeEditorProvider implements INotebookEditorProvider, IAsyncDisp // Possible we have a git diff view (with two editors git and file scheme), and we open the file view // on the side (different view column). const gitSchemeEditor = this.documentManager.visibleTextEditors.find(editorUri => - editorUri.document.uri.scheme === 'git' && - this.fileSystem.arePathsSame(editorUri.document.uri.fsPath, editor.document.uri.fsPath)); + editorUri.document.uri.scheme === 'git' && + this.fileSystem.arePathsSame(editorUri.document.uri.fsPath, editor.document.uri.fsPath)); - if (!gitSchemeEditor){ + if (!gitSchemeEditor) { return false; } const fileSchemeEditor = this.documentManager.visibleTextEditors.find(editorUri => - editorUri.document.uri.scheme === 'file' && - this.fileSystem.arePathsSame(editorUri.document.uri.fsPath, editor.document.uri.fsPath) && - editorUri.viewColumn === gitSchemeEditor.viewColumn); - if (!fileSchemeEditor){ + editorUri.document.uri.scheme === 'file' && + this.fileSystem.arePathsSame(editorUri.document.uri.fsPath, editor.document.uri.fsPath) && + editorUri.viewColumn === gitSchemeEditor.viewColumn); + if (!fileSchemeEditor) { return false; } diff --git a/src/client/datascience/jupyter/jupyterExporter.ts b/src/client/datascience/jupyter/jupyterExporter.ts index 89fed8dfbbf5..7f8f5f8a3203 100644 --- a/src/client/datascience/jupyter/jupyterExporter.ts +++ b/src/client/datascience/jupyter/jupyterExporter.ts @@ -63,12 +63,13 @@ export class JupyterExporter implements INotebookExporter { const matcher = new CellMatcher(this.configService.getSettings().datascience); // Combine this into a JSON object - return { + const obj = { cells: this.pruneCells(cells, matcher), nbformat: 4, nbformat_minor: 2, metadata: metadata }; + return obj; } // For exporting, put in a cell that will change the working directory back to the workspace directory so relative data paths will load correctly @@ -89,7 +90,8 @@ export class JupyterExporter implements INotebookExporter { id: uuid(), file: Identifiers.EmptyFileName, line: 0, - state: CellState.finished + state: CellState.finished, + executedInCurrentKernel: false }; return [cell, ...cells]; diff --git a/src/client/datascience/jupyter/jupyterImporter.ts b/src/client/datascience/jupyter/jupyterImporter.ts index b0b644207634..b172db22f82f 100644 --- a/src/client/datascience/jupyter/jupyterImporter.ts +++ b/src/client/datascience/jupyter/jupyterImporter.ts @@ -103,6 +103,7 @@ export class JupyterImporter implements INotebookImporter { file: Identifiers.EmptyFileName, line: 0, state: CellState.finished, + executedInCurrentKernel: false, data: c, type: 'preview' }; diff --git a/src/client/datascience/jupyter/jupyterNotebook.ts b/src/client/datascience/jupyter/jupyterNotebook.ts index 94bcf6186119..4e76734e9d0e 100644 --- a/src/client/datascience/jupyter/jupyterNotebook.ts +++ b/src/client/datascience/jupyter/jupyterNotebook.ts @@ -301,7 +301,8 @@ export class JupyterNotebookBase implements INotebook { id: uuid(), file: '', line: 0, - state: CellState.finished + state: CellState.finished, + executedInCurrentKernel: true }; } diff --git a/src/client/datascience/types.ts b/src/client/datascience/types.ts index 1e83d0cc5a3f..21f691caa702 100644 --- a/src/client/datascience/types.ts +++ b/src/client/datascience/types.ts @@ -244,6 +244,7 @@ export interface INotebookEditorProvider { show(file: Uri): Promise; createNew(): Promise; getNotebookOptions(): Promise; + getNextNewNotebookUri(): Promise; } // For native editing, the INotebookEditor acts like a TextEditor and a TextDocument together @@ -351,6 +352,7 @@ export interface ICell { file: string; line: number; state: CellState; + executedInCurrentKernel: boolean; data: nbformat.ICodeCell | nbformat.IRawCell | nbformat.IMarkdownCell | IMessageCell; extraLines?: number[]; } diff --git a/src/datascience-ui/interactive-common/mainState.ts b/src/datascience-ui/interactive-common/mainState.ts index fa218535e1dc..d423c5f8c21a 100644 --- a/src/datascience-ui/interactive-common/mainState.ts +++ b/src/datascience-ui/interactive-common/mainState.ts @@ -135,7 +135,8 @@ export function createEmptyCell(id: string | undefined, executionCount: number | id: id ? id : Identifiers.EditCellId, file: Identifiers.EmptyFileName, line: 0, - state: CellState.finished + state: CellState.finished, + executedInCurrentKernel: false }; } @@ -217,6 +218,7 @@ export function generateCells(filePath: string, repetitions: number): ICell[] { file: path.join(filePath, 'foo.py').toLowerCase(), line: 1, state: key === cellData.length - 1 ? CellState.executing : CellState.finished, + executedInCurrentKernel: false, type: key === 3 ? 'preview' : 'execute', data: data }; diff --git a/src/datascience-ui/native-editor/nativeCell.tsx b/src/datascience-ui/native-editor/nativeCell.tsx index 214bb23c4c3e..38d328b88c21 100644 --- a/src/datascience-ui/native-editor/nativeCell.tsx +++ b/src/datascience-ui/native-editor/nativeCell.tsx @@ -574,7 +574,10 @@ export class NativeCell extends React.Component { this.props.stateController.sendCommand(NativeCommandType.RunBelow, 'mouse'); }; const gatherDisabled = () => { - return this.props.cellVM.cell.data.execution_count === null || this.props.cellVM.cell.data.cell_type !== 'code' || getSettings().enableGather === false; + return this.props.cellVM.cell.data.execution_count === null || + this.props.cellVM.cell.executedInCurrentKernel === false || + this.props.cellVM.cell.data.cell_type !== 'code' || + getSettings().enableGather === false; }; const canRunAbove = this.props.stateController.canRunAbove(cellId); const canRunBelow = this.props.cellVM.cell.state === CellState.finished || this.props.cellVM.cell.state === CellState.error; diff --git a/src/test/datascience/editor-integration/cellhashprovider.unit.test.ts b/src/test/datascience/editor-integration/cellhashprovider.unit.test.ts index 77fd24f7eea6..938d83c51402 100644 --- a/src/test/datascience/editor-integration/cellhashprovider.unit.test.ts +++ b/src/test/datascience/editor-integration/cellhashprovider.unit.test.ts @@ -68,7 +68,8 @@ suite('CellHashProvider Unit Tests', () => { execution_count: 1 }, id: '1', - state: CellState.init + state: CellState.init, + executedInCurrentKernel: false }; return hashProvider.preExecute(cell, false); } diff --git a/src/test/datascience/gather/gather.unit.test.ts b/src/test/datascience/gather/gather.unit.test.ts index 81c23eb66fa2..c218fb31af6b 100644 --- a/src/test/datascience/gather/gather.unit.test.ts +++ b/src/test/datascience/gather/gather.unit.test.ts @@ -17,6 +17,7 @@ suite('DataScience code gathering unit tests', () => { file: '2DB9B899-6519-4E1B-88B0-FA728A274115', line: 0, state: 2, + executedInCurrentKernel: true, data: { source: `from bokeh.plotting import show, figure, output_notebook\noutput_notebook()`, cell_type: 'code', @@ -30,6 +31,7 @@ suite('DataScience code gathering unit tests', () => { file: '2DB9B899-6519-4E1B-88B0-FA728A274115', line: 0, state: 2, + executedInCurrentKernel: true, data: { source: `x = [1,2,3,4,5]\ny = [21,9,15,17,4]\nprint('This is some irrelevant code')`, cell_type: 'code', @@ -43,6 +45,7 @@ suite('DataScience code gathering unit tests', () => { file: '2DB9B899-6519-4E1B-88B0-FA728A274115', line: 0, state: 2, + executedInCurrentKernel: true, data: { source: `p=figure(title='demo',x_axis_label='x',y_axis_label='y')`, cell_type: 'code', @@ -56,6 +59,7 @@ suite('DataScience code gathering unit tests', () => { file: '2DB9B899-6519-4E1B-88B0-FA728A274115', line: 0, state: 2, + executedInCurrentKernel: true, data: { source: 'p.line(x,y,line_width=2)', cell_type: 'code', @@ -69,6 +73,7 @@ suite('DataScience code gathering unit tests', () => { file: '2DB9B899-6519-4E1B-88B0-FA728A274115', line: 0, state: 2, + executedInCurrentKernel: true, data: { source: 'show(p)', cell_type: 'code', diff --git a/src/test/datascience/jupyterVariables.unit.test.ts b/src/test/datascience/jupyterVariables.unit.test.ts index 5efba16b5190..f6789568a8d8 100644 --- a/src/test/datascience/jupyterVariables.unit.test.ts +++ b/src/test/datascience/jupyterVariables.unit.test.ts @@ -35,7 +35,8 @@ suite('JupyterVariables', () => { id: '0', file: '', line: 0, - state: CellState.finished + state: CellState.finished, + executedInCurrentKernel: false }; } diff --git a/src/test/datascience/testNativeEditorProvider.ts b/src/test/datascience/testNativeEditorProvider.ts index 5a92a275b17c..d9e2b7b07733 100644 --- a/src/test/datascience/testNativeEditorProvider.ts +++ b/src/test/datascience/testNativeEditorProvider.ts @@ -95,4 +95,8 @@ export class TestNativeEditorProvider implements INotebookEditorProvider { public async getNotebookOptions(): Promise { return this.realProvider.getNotebookOptions(); } + + public async getNextNewNotebookUri(): Promise { + return Uri.file(''); + } } From af781c10ce16c001034032d89b50822cf0c5992a Mon Sep 17 00:00:00 2001 From: Jim Griesmer Date: Mon, 28 Oct 2019 01:41:20 -0700 Subject: [PATCH 09/42] Bug fix and show gathered notebook --- src/client/datascience/cellFactory.ts | 25 ++++---- src/client/datascience/gather/gather.ts | 2 +- .../datascience/gather/gatherListener.ts | 58 ++++++------------- 3 files changed, 34 insertions(+), 51 deletions(-) diff --git a/src/client/datascience/cellFactory.ts b/src/client/datascience/cellFactory.ts index 662d888666d3..a833380799ea 100644 --- a/src/client/datascience/cellFactory.ts +++ b/src/client/datascience/cellFactory.ts @@ -109,28 +109,29 @@ export function hasCells(document: TextDocument, settings?: IDataScienceSettings return false; } -export function generateCellRangesFromString(source: string, settings?: IDataScienceSettings): { range: Range; title: string; cell_type: string }[] { +function generateCellRangesFromStringArray(lines: string[], settings?: IDataScienceSettings): { range: Range; title: string; cell_type: string }[] { const matcher = new CellMatcher(settings); const cells: { range: Range; title: string; cell_type: string }[] = []; - const lines: string[] = source.splitLines({ trim: true, removeEmptyEntries: false }); for (let index = 0; index < lines.length; index += 1) { if (matcher.isCell(lines[index])) { // We have a cell, find the next cell - let j = index + 1; - while (j < lines.length && !matcher.isCell(lines[j])) { - j += 1; + let endCellIndex = index + 1; + while (endCellIndex < lines.length && !matcher.isCell(lines[endCellIndex])) { + endCellIndex += 1; } + endCellIndex -= 1; // Get last line of cell. const result = matcher.exec(lines[index]); if (result !== undefined) { cells.push({ - range: new Range(index, 0, j - 1, lines[j - 1].length), + range: new Range(index + 1, 0, endCellIndex, lines[endCellIndex].length), title: result, cell_type: matcher.getCellType(lines[index]) }); } + index = endCellIndex; } } @@ -138,20 +139,22 @@ export function generateCellRangesFromString(source: string, settings?: IDataSci } export function generateCellsFromString(source: string, settings?: IDataScienceSettings): ICell[] { - // Get our ranges. They'll determine our cells - const ranges = generateCellRangesFromString(source, settings); + const lines: string[] = source.splitLines({ trim: false, removeEmptyEntries: false }); - const lines: string[] = source.splitLines(); + // Get our ranges. They'll determine our cells + const ranges = generateCellRangesFromStringArray(lines, settings); // For each one, get its text and turn it into a cell return Array.prototype.concat(...ranges.map(r => { - const code = lines.slice(r.range.start.line, r.range.end.line).join(''); + const code = lines.slice(r.range.start.line, r.range.end.line).join('\n'); return generateCells(settings, code, '', r.range.start.line, false, uuid()); })); } export function generateCellRangesFromDocument(document: TextDocument, settings?: IDataScienceSettings): { range: Range; title: string; cell_type: string }[] { - return generateCellRangesFromString(document.getText(), settings); + const lines: string[] = document.getText().splitLines({ trim: false, removeEmptyEntries: false }); + + return generateCellRangesFromStringArray(lines, settings); } export function generateCellsFromDocument(document: TextDocument, settings?: IDataScienceSettings): ICell[] { diff --git a/src/client/datascience/gather/gather.ts b/src/client/datascience/gather/gather.ts index 317ce423222e..047781e200fb 100644 --- a/src/client/datascience/gather/gather.ts +++ b/src/client/datascience/gather/gather.ts @@ -91,7 +91,7 @@ export class GatherExecution implements IGatherExecution, INotebookExecutionLogg const program = slices.length > 0 ? slices[0].cellSlices.reduce(concat, '').replace(/#%%/g, defaultCellMarker) : ''; // Add a comment at the top of the file explaining what gather does - const descriptor = '# This file contains the minimal amount of code required to produce the code cell you gathered.\n'; + const descriptor = '# This file contains the minimal amount of code required to produce the code cell gathered.\n'; return descriptor.concat(program); } diff --git a/src/client/datascience/gather/gatherListener.ts b/src/client/datascience/gather/gatherListener.ts index 6849f9138180..ebd2c83cb95a 100644 --- a/src/client/datascience/gather/gatherListener.ts +++ b/src/client/datascience/gather/gatherListener.ts @@ -1,8 +1,7 @@ import { inject, injectable } from 'inversify'; -import { Event, EventEmitter, Position, ViewColumn } from 'vscode'; -import { IApplicationShell, IDocumentManager } from '../../common/application/types'; -import { PYTHON_LANGUAGE } from '../../common/constants'; -import { IFileSystem } from '../../common/platform/types'; +import * as uuid from 'uuid/v4'; +import { Event, EventEmitter } from 'vscode'; +import { IApplicationShell } from '../../common/application/types'; import { noop } from '../../common/utils/misc'; import { generateCellsFromString } from '../cellFactory'; import { InteractiveWindowMessages } from '../interactive-common/interactiveWindowTypes'; @@ -16,10 +15,8 @@ export class GatherListener implements IInteractiveWindowListener { constructor( @inject(INotebookExporter) private jupyterExporter: INotebookExporter, @inject(INotebookEditorProvider) private ipynbProvider: INotebookEditorProvider, - @inject(IDocumentManager) private documentManager: IDocumentManager, @inject(IApplicationShell) private applicationShell: IApplicationShell, - @inject(IGatherExecution) private gatherExecution: IGatherExecution, - @inject(IFileSystem) private fileSystem: IFileSystem) { } + @inject(IGatherExecution) private gatherExecution: IGatherExecution) { } public dispose() { noop(); @@ -57,44 +54,27 @@ export class GatherListener implements IInteractiveWindowListener { private gatherCodeInternal = async (cell: ICell) => { const slicedProgram = this.gatherExecution.gatherCode(cell); + const newuri = await this.ipynbProvider.getNextNewNotebookUri(); - // Don't want to open the gathered code on top of the interactive window - let viewColumn: ViewColumn | undefined; - const fileNameMatch = this.documentManager.visibleTextEditors.filter(textEditor => this.fileSystem.arePathsSame(textEditor.document.fileName, cell.file)); - const definedVisibleEditors = this.documentManager.visibleTextEditors.filter(textEditor => textEditor.viewColumn !== undefined); - if (this.documentManager.visibleTextEditors.length > 0 && fileNameMatch.length > 0) { - // Original file is visible - viewColumn = fileNameMatch[0].viewColumn; - } else if (this.documentManager.visibleTextEditors.length > 0 && definedVisibleEditors.length > 0) { - // There is a visible text editor, just not the original file. Make sure viewColumn isn't undefined - viewColumn = definedVisibleEditors[0].viewColumn; - } else { - // Only one panel open and interactive window is occupying it, or original file is open but hidden - viewColumn = ViewColumn.Beside; - } + let cells: ICell[] = [{ + id: uuid(), + file: '', + line: 0, + state: 0, + executedInCurrentKernel: false, + data: { + cell_type: 'markdown', + source: '## This notebook was generated for a gathered cell.', + metadata: {} + } + }]; // Create new notebook with the returned program and open it. - const cells: ICell[] = generateCellsFromString(slicedProgram); - if (cells.length === 0) { - return; - } + cells = cells.concat(generateCellsFromString(slicedProgram)); const notebook = await this.jupyterExporter.translateToNotebook(cells); const contents = JSON.stringify(notebook); - const newuri = await this.ipynbProvider.getNextNewNotebookUri(); - await (await this.ipynbProvider.open(newuri, contents)).show(); - - // Create a new open editor with the returned program in the right panel - const doc = await this.documentManager.openTextDocument({ - content: slicedProgram, - language: PYTHON_LANGUAGE - }); - const editor = await this.documentManager.showTextDocument(doc, viewColumn); - - // Edit the document so that it is dirty (add a space at the end) - editor.edit((editBuilder) => { - editBuilder.insert(new Position(editor.document.lineCount, 0), '\n'); - }); + await this.ipynbProvider.open(newuri, contents); } } From 4e2d3051714a3120db88c092ffbcabd505de1d4d Mon Sep 17 00:00:00 2001 From: Jim Griesmer Date: Mon, 28 Oct 2019 10:51:59 -0700 Subject: [PATCH 10/42] Make strings localizable --- package.nls.json | 4 +++- src/client/common/utils/localize.ts | 2 ++ src/client/datascience/gather/gather.ts | 3 +-- src/client/datascience/gather/gatherListener.ts | 3 ++- 4 files changed, 8 insertions(+), 4 deletions(-) diff --git a/package.nls.json b/package.nls.json index 214b992680d4..9eb2f7d4f72a 100644 --- a/package.nls.json +++ b/package.nls.json @@ -378,5 +378,7 @@ "DataScience.notebookNotFound" : "python -m jupyter notebook --version is not running", "DataScience.findJupyterCommandProgress" : "Active interpreter does not support {0}. Searching for the best available interpreter.", "DataScience.findJupyterCommandProgressCheckInterpreter": "Checking {0}.", - "DataScience.findJupyterCommandProgressSearchCurrentPath": "Searching current path." + "DataScience.findJupyterCommandProgressSearchCurrentPath": "Searching current path.", + "DataScience.gatheredScriptDescription": "# This file contains the minimal amount of code required to produce the code cell gathered.\n", + "DataScience.gatheredNotebookDescriptionInMarkdown": "## This notebook was generated for a gathered cell." } diff --git a/src/client/common/utils/localize.ts b/src/client/common/utils/localize.ts index da6e41fb2c5e..aa5d51b9c8cc 100644 --- a/src/client/common/utils/localize.ts +++ b/src/client/common/utils/localize.ts @@ -294,6 +294,8 @@ export namespace DataScience { export const findJupyterCommandProgress = localize('DataScience.findJupyterCommandProgress', 'Active interpreter does not support {0}. Searching for the best available interpreter.'); export const findJupyterCommandProgressCheckInterpreter = localize('DataScience.findJupyterCommandProgressCheckInterpreter', 'Checking {0}.'); export const findJupyterCommandProgressSearchCurrentPath = localize('DataScience.findJupyterCommandProgressSearchCurrentPath', 'Searching current path.'); + export const gatheredScriptDescription = localize('DataScience.gatheredScriptDescription', '# This file contains the minimal amount of code required to produce the code cell gathered.\n'); + export const gatheredNotebookDescriptionInMarkdown = localize('DataScience.gatheredNotebookDescriptionInMarkdown', '## This notebook was generated for a gathered cell.'); } export namespace DebugConfigStrings { diff --git a/src/client/datascience/gather/gather.ts b/src/client/datascience/gather/gather.ts index 047781e200fb..680e1ba35559 100644 --- a/src/client/datascience/gather/gather.ts +++ b/src/client/datascience/gather/gather.ts @@ -91,7 +91,7 @@ export class GatherExecution implements IGatherExecution, INotebookExecutionLogg const program = slices.length > 0 ? slices[0].cellSlices.reduce(concat, '').replace(/#%%/g, defaultCellMarker) : ''; // Add a comment at the top of the file explaining what gather does - const descriptor = '# This file contains the minimal amount of code required to produce the code cell gathered.\n'; + const descriptor = localize.DataScience.gatheredScriptDescription(); return descriptor.concat(program); } @@ -125,7 +125,6 @@ export class GatherExecution implements IGatherExecution, INotebookExecutionLogg * Accumulator to concatenate cell slices for a sliced program, preserving cell structures. */ function concat(existingText: string, newText: CellSlice): string { - // tslint:disable-next-line: jsdoc-format // Include our cell marker so that cell slices are preserved return `${existingText}#%%\n${newText.textSliceLines}\n\n`; } diff --git a/src/client/datascience/gather/gatherListener.ts b/src/client/datascience/gather/gatherListener.ts index ebd2c83cb95a..0596fed01eaa 100644 --- a/src/client/datascience/gather/gatherListener.ts +++ b/src/client/datascience/gather/gatherListener.ts @@ -2,6 +2,7 @@ import { inject, injectable } from 'inversify'; import * as uuid from 'uuid/v4'; import { Event, EventEmitter } from 'vscode'; import { IApplicationShell } from '../../common/application/types'; +import * as localize from '../../common/utils/localize'; import { noop } from '../../common/utils/misc'; import { generateCellsFromString } from '../cellFactory'; import { InteractiveWindowMessages } from '../interactive-common/interactiveWindowTypes'; @@ -64,7 +65,7 @@ export class GatherListener implements IInteractiveWindowListener { executedInCurrentKernel: false, data: { cell_type: 'markdown', - source: '## This notebook was generated for a gathered cell.', + source: localize.DataScience.gatheredNotebookDescriptionInMarkdown(), metadata: {} } }]; From 4a01d0a5e447b668c0a716c8d48c74f995344a10 Mon Sep 17 00:00:00 2001 From: Jim Griesmer Date: Mon, 28 Oct 2019 17:17:20 -0700 Subject: [PATCH 11/42] executedInCurrentKernal => executeKernalId --- src/client/datascience/cellFactory.ts | 4 ++-- src/client/datascience/gather/gather.ts | 1 - src/client/datascience/gather/gatherListener.ts | 2 +- .../interactive-common/interactiveBase.ts | 5 ++--- .../interactive-common/interactiveWindowTypes.ts | 2 +- .../interactive-ipynb/nativeEditor.ts | 16 ++++++++++------ .../datascience/jupyter/jupyterExporter.ts | 5 ++--- .../datascience/jupyter/jupyterImporter.ts | 2 +- .../datascience/jupyter/jupyterKernelSpec.ts | 2 +- .../datascience/jupyter/jupyterNotebook.ts | 6 +++++- src/client/datascience/jupyter/jupyterSession.ts | 6 ++++++ .../jupyter/liveshare/guestJupyterNotebook.ts | 4 ++++ src/client/datascience/types.ts | 4 +++- .../interactive-common/mainState.ts | 5 +++-- .../interactive-common/mainStateController.ts | 5 +++-- src/datascience-ui/native-editor/nativeCell.tsx | 5 ++++- .../cellhashprovider.unit.test.ts | 2 +- src/test/datascience/execution.unit.test.ts | 10 +++++++--- src/test/datascience/gather/gather.unit.test.ts | 10 +++++----- .../datascience/jupyterVariables.unit.test.ts | 2 +- src/test/datascience/mockJupyterSession.ts | 8 ++++++++ 21 files changed, 70 insertions(+), 36 deletions(-) diff --git a/src/client/datascience/cellFactory.ts b/src/client/datascience/cellFactory.ts index a833380799ea..38622c85303f 100644 --- a/src/client/datascience/cellFactory.ts +++ b/src/client/datascience/cellFactory.ts @@ -44,7 +44,7 @@ function generateCodeCell(code: string[], file: string, line: number, id: string file: file, line: line, state: CellState.init, - executedInCurrentKernel: false + executeKernelId: undefined }; } @@ -55,7 +55,7 @@ function generateMarkdownCell(code: string[], file: string, line: number, id: st file: file, line: line, state: CellState.finished, - executedInCurrentKernel: false, + executeKernelId: undefined, data: { cell_type: 'markdown', source: generateMarkdownFromCodeLines(code), diff --git a/src/client/datascience/gather/gather.ts b/src/client/datascience/gather/gather.ts index 680e1ba35559..e2f64e62d5e4 100644 --- a/src/client/datascience/gather/gather.ts +++ b/src/client/datascience/gather/gather.ts @@ -77,7 +77,6 @@ export class GatherExecution implements IGatherExecution, INotebookExecutionLogg * For a given code cell, returns a string representing a program containing all the code it depends on. */ public gatherCode(vscCell: IVscCell): string { - // sliceAllExecutions does a lookup based on executionEventId const gatherCell = convertVscToGatherCell(vscCell); if (gatherCell === undefined) { return ''; diff --git a/src/client/datascience/gather/gatherListener.ts b/src/client/datascience/gather/gatherListener.ts index 0596fed01eaa..09021acb3505 100644 --- a/src/client/datascience/gather/gatherListener.ts +++ b/src/client/datascience/gather/gatherListener.ts @@ -62,7 +62,7 @@ export class GatherListener implements IInteractiveWindowListener { file: '', line: 0, state: 0, - executedInCurrentKernel: false, + executeKernelId: undefined, data: { cell_type: 'markdown', source: localize.DataScience.gatheredNotebookDescriptionInMarkdown(), diff --git a/src/client/datascience/interactive-common/interactiveBase.ts b/src/client/datascience/interactive-common/interactiveBase.ts index 4e80799be373..b4ba0896c6b5 100644 --- a/src/client/datascience/interactive-common/interactiveBase.ts +++ b/src/client/datascience/interactive-common/interactiveBase.ts @@ -561,7 +561,7 @@ export abstract class InteractiveBase extends WebViewHostthis.getNotebook()).getKernelId() : undefined } ]); @@ -417,6 +418,12 @@ export class NativeEditor extends InteractiveBase implements INotebookEditor { cells.forEach(c => { const index = this.visibleCells.findIndex(v => v.id === c.id); this.visibleCells[index] = c; + if (c.state === CellState.executing) { + const n = this.getNotebook(); + if (n !== undefined) { + c.executeKernelId = n.getKernelId(); + } + } }); // Indicate dirty @@ -516,7 +523,7 @@ export class NativeEditor extends InteractiveBase implements INotebookEditor { file: Identifiers.EmptyFileName, line: 0, state: CellState.finished, - executedInCurrentKernel: false, + executeKernelId: undefined, data: c }; }), forceDirty); @@ -531,7 +538,7 @@ export class NativeEditor extends InteractiveBase implements INotebookEditor { line: 0, file: Identifiers.EmptyFileName, state: CellState.finished, - executedInCurrentKernel: false, + executeKernelId: undefined, data: { cell_type: 'code', outputs: [], @@ -857,9 +864,6 @@ export class NativeEditor extends InteractiveBase implements INotebookEditor { } private onRestartKernel() { - if (this.cells.length > 0) { - this.cells.forEach((c) => { c.executedInCurrentKernel = false; }); - } this.sendCellsToWebView(this.cells); } } diff --git a/src/client/datascience/jupyter/jupyterExporter.ts b/src/client/datascience/jupyter/jupyterExporter.ts index 7f8f5f8a3203..904daf2aff2d 100644 --- a/src/client/datascience/jupyter/jupyterExporter.ts +++ b/src/client/datascience/jupyter/jupyterExporter.ts @@ -63,13 +63,12 @@ export class JupyterExporter implements INotebookExporter { const matcher = new CellMatcher(this.configService.getSettings().datascience); // Combine this into a JSON object - const obj = { + return { cells: this.pruneCells(cells, matcher), nbformat: 4, nbformat_minor: 2, metadata: metadata }; - return obj; } // For exporting, put in a cell that will change the working directory back to the workspace directory so relative data paths will load correctly @@ -91,7 +90,7 @@ export class JupyterExporter implements INotebookExporter { file: Identifiers.EmptyFileName, line: 0, state: CellState.finished, - executedInCurrentKernel: false + executeKernelId: undefined }; return [cell, ...cells]; diff --git a/src/client/datascience/jupyter/jupyterImporter.ts b/src/client/datascience/jupyter/jupyterImporter.ts index b172db22f82f..4455c1b41b95 100644 --- a/src/client/datascience/jupyter/jupyterImporter.ts +++ b/src/client/datascience/jupyter/jupyterImporter.ts @@ -103,7 +103,7 @@ export class JupyterImporter implements INotebookImporter { file: Identifiers.EmptyFileName, line: 0, state: CellState.finished, - executedInCurrentKernel: false, + executeKernelId: undefined, data: c, type: 'preview' }; diff --git a/src/client/datascience/jupyter/jupyterKernelSpec.ts b/src/client/datascience/jupyter/jupyterKernelSpec.ts index 9881db8671c7..25b171e0464c 100644 --- a/src/client/datascience/jupyter/jupyterKernelSpec.ts +++ b/src/client/datascience/jupyter/jupyterKernelSpec.ts @@ -15,7 +15,7 @@ export class JupyterKernelSpec implements IJupyterKernelSpec { public language: string; public path: string; public specFile: string | undefined; - constructor(specModel : Kernel.ISpecModel, file?: string) { + constructor(specModel: Kernel.ISpecModel, file?: string) { this.name = specModel.name; this.language = specModel.language; this.path = specModel.argv && specModel.argv.length > 0 ? specModel.argv[0] : ''; diff --git a/src/client/datascience/jupyter/jupyterNotebook.ts b/src/client/datascience/jupyter/jupyterNotebook.ts index 4e76734e9d0e..1bc33056f329 100644 --- a/src/client/datascience/jupyter/jupyterNotebook.ts +++ b/src/client/datascience/jupyter/jupyterNotebook.ts @@ -273,6 +273,10 @@ export class JupyterNotebookBase implements INotebook { }); } + public getKernelId(): string | undefined { + return this.session.getKernelId(); + } + public async getSysInfo(): Promise { // tslint:disable-next-line:no-multiline-string const versionCells = await this.executeSilently(`import sys\r\nsys.version`); @@ -302,7 +306,7 @@ export class JupyterNotebookBase implements INotebook { file: '', line: 0, state: CellState.finished, - executedInCurrentKernel: true + executeKernelId: this.getKernelId() }; } diff --git a/src/client/datascience/jupyter/jupyterSession.ts b/src/client/datascience/jupyter/jupyterSession.ts index 45d68a528c65..e57c43edc617 100644 --- a/src/client/datascience/jupyter/jupyterSession.ts +++ b/src/client/datascience/jupyter/jupyterSession.ts @@ -47,6 +47,12 @@ export class JupyterSession implements IJupyterSession { return this.shutdown(); } + public getKernelId(): string | undefined { + if (this.session !== undefined) { + return this.session.id; + } + } + public async shutdown(): Promise { // Destroy the notebook file if not local. Local is cleaned up when we destroy the kernel spec. if (this.notebookFiles.length && this.contentsManager && this.connInfo && !this.connInfo.localLaunch) { diff --git a/src/client/datascience/jupyter/liveshare/guestJupyterNotebook.ts b/src/client/datascience/jupyter/liveshare/guestJupyterNotebook.ts index d2c8a572f6f2..f7fe9e588881 100644 --- a/src/client/datascience/jupyter/liveshare/guestJupyterNotebook.ts +++ b/src/client/datascience/jupyter/liveshare/guestJupyterNotebook.ts @@ -55,6 +55,10 @@ export class GuestJupyterNotebook return Promise.resolve(); } + public getKernelId(): string | undefined { + return 'NA'; + } + public clear(_id: string): void { // We don't do anything as we don't cache results in this class. noop(); diff --git a/src/client/datascience/types.ts b/src/client/datascience/types.ts index 21f691caa702..1c5c46eea66f 100644 --- a/src/client/datascience/types.ts +++ b/src/client/datascience/types.ts @@ -96,6 +96,7 @@ export interface INotebook extends IAsyncDisposable { waitForIdle(timeoutInMs: number): Promise; interruptKernel(timeoutInMs: number): Promise; setLaunchingFile(file: string): Promise; + getKernelId(): string | undefined; getSysInfo(): Promise; setMatplotLibStyle(useDark: boolean): Promise; } @@ -161,6 +162,7 @@ export interface IJupyterPasswordConnect { export const IJupyterSession = Symbol('IJupyterSession'); export interface IJupyterSession extends IAsyncDisposable { onRestarted: Event; + getKernelId(): string | undefined; restart(timeout: number): Promise; interrupt(timeout: number): Promise; waitForIdle(timeout: number): Promise; @@ -352,7 +354,7 @@ export interface ICell { file: string; line: number; state: CellState; - executedInCurrentKernel: boolean; + executeKernelId: string | undefined; data: nbformat.ICodeCell | nbformat.IRawCell | nbformat.IMarkdownCell | IMessageCell; extraLines?: number[]; } diff --git a/src/datascience-ui/interactive-common/mainState.ts b/src/datascience-ui/interactive-common/mainState.ts index d423c5f8c21a..549eed955fbe 100644 --- a/src/datascience-ui/interactive-common/mainState.ts +++ b/src/datascience-ui/interactive-common/mainState.ts @@ -61,6 +61,7 @@ export interface IMainState { isAtBottom: boolean; newCellId?: string; loadTotal?: number; + kernelId?: string; } export interface IFont { @@ -136,7 +137,7 @@ export function createEmptyCell(id: string | undefined, executionCount: number | file: Identifiers.EmptyFileName, line: 0, state: CellState.finished, - executedInCurrentKernel: false + executeKernelId: undefined }; } @@ -218,7 +219,7 @@ export function generateCells(filePath: string, repetitions: number): ICell[] { file: path.join(filePath, 'foo.py').toLowerCase(), line: 1, state: key === cellData.length - 1 ? CellState.executing : CellState.finished, - executedInCurrentKernel: false, + executeKernelId: undefined, type: key === 3 ? 'preview' : 'execute', data: data }; diff --git a/src/datascience-ui/interactive-common/mainStateController.ts b/src/datascience-ui/interactive-common/mainStateController.ts index c17515b9e5b0..d97c5b267073 100644 --- a/src/datascience-ui/interactive-common/mainStateController.ts +++ b/src/datascience-ui/interactive-common/mainStateController.ts @@ -224,7 +224,7 @@ export class MainStateController implements IMessageHandler { case InteractiveWindowMessages.RestartKernel: // Go through all vms that are currently executing and mark them as finished - this.handleRestarted(); + this.handleRestarted(payload); break; case InteractiveWindowMessages.StartDebugging: @@ -966,7 +966,7 @@ export class MainStateController implements IMessageHandler { } } - private handleRestarted() { + private handleRestarted(payload: string | undefined) { this.suspendUpdates(); // When we restart, make sure to turn off all executing cells. They aren't executing anymore @@ -982,6 +982,7 @@ export class MainStateController implements IMessageHandler { this.setState({ cellVMs: newVMs }); } this.setState({ currentExecutionCount: 0 }); + this.setState({ kernelId: payload }); this.resumeUpdates(); // Update our variables diff --git a/src/datascience-ui/native-editor/nativeCell.tsx b/src/datascience-ui/native-editor/nativeCell.tsx index 38d328b88c21..0e0b265fb9f8 100644 --- a/src/datascience-ui/native-editor/nativeCell.tsx +++ b/src/datascience-ui/native-editor/nativeCell.tsx @@ -575,7 +575,10 @@ export class NativeCell extends React.Component { }; const gatherDisabled = () => { return this.props.cellVM.cell.data.execution_count === null || - this.props.cellVM.cell.executedInCurrentKernel === false || + this.props.cellVM.cell.executeKernelId === null || + this.props.cellVM.cell.executeKernelId === undefined || + this.props.stateController.getState().kernelId === null || + this.props.cellVM.cell.executeKernelId !== this.props.stateController.getState().kernelId || this.props.cellVM.cell.data.cell_type !== 'code' || getSettings().enableGather === false; }; diff --git a/src/test/datascience/editor-integration/cellhashprovider.unit.test.ts b/src/test/datascience/editor-integration/cellhashprovider.unit.test.ts index 938d83c51402..646ecedc8269 100644 --- a/src/test/datascience/editor-integration/cellhashprovider.unit.test.ts +++ b/src/test/datascience/editor-integration/cellhashprovider.unit.test.ts @@ -69,7 +69,7 @@ suite('CellHashProvider Unit Tests', () => { }, id: '1', state: CellState.init, - executedInCurrentKernel: false + executeKernelId: undefined }; return hashProvider.preExecute(cell, false); } diff --git a/src/test/datascience/execution.unit.test.ts b/src/test/datascience/execution.unit.test.ts index 65e7fe4497c7..38defebbabbd 100644 --- a/src/test/datascience/execution.unit.test.ts +++ b/src/test/datascience/execution.unit.test.ts @@ -107,6 +107,10 @@ class MockJupyterNotebook implements INotebook { return Promise.resolve(undefined); } + public getKernelId(): string | undefined { + return 'fake_kernel_id'; + } + public interruptKernel(_timeout: number): Promise { throw new Error('Method not implemented'); } @@ -573,7 +577,7 @@ suite('Jupyter Execution', async () => { when(workspaceService.onDidChangeConfiguration).thenReturn(configChangeEvent.event); when(application.withProgress(anything(), anything())).thenCall((_, cb: (_: any, token: any) => Promise) => { return new Promise((resolve, reject) => { - cb({report: noop}, new CancellationTokenSource().token).then(resolve).catch(reject); + cb({ report: noop }, new CancellationTokenSource().token).then(resolve).catch(reject); }); }); @@ -751,7 +755,7 @@ suite('Jupyter Execution', async () => { reset(application); when(application.withProgress(anything(), anything())).thenCall((_, cb: (_: any, token: any) => Promise) => { return new Promise((resolve, reject) => { - cb({report: noop}, progressCancellation.token).then(resolve).catch(reject); + cb({ report: noop }, progressCancellation.token).then(resolve).catch(reject); }); }); @@ -770,7 +774,7 @@ suite('Jupyter Execution', async () => { reset(application); when(application.withProgress(anything(), anything())).thenCall((_, cb: (_: any, token: any) => Promise) => { return new Promise((resolve, reject) => { - cb({report: noop}, progressCancellation.token).then(resolve).catch(reject); + cb({ report: noop }, progressCancellation.token).then(resolve).catch(reject); }); }); diff --git a/src/test/datascience/gather/gather.unit.test.ts b/src/test/datascience/gather/gather.unit.test.ts index c218fb31af6b..da9754d5c1c9 100644 --- a/src/test/datascience/gather/gather.unit.test.ts +++ b/src/test/datascience/gather/gather.unit.test.ts @@ -17,7 +17,7 @@ suite('DataScience code gathering unit tests', () => { file: '2DB9B899-6519-4E1B-88B0-FA728A274115', line: 0, state: 2, - executedInCurrentKernel: true, + executeKernelId: 'true', data: { source: `from bokeh.plotting import show, figure, output_notebook\noutput_notebook()`, cell_type: 'code', @@ -31,7 +31,7 @@ suite('DataScience code gathering unit tests', () => { file: '2DB9B899-6519-4E1B-88B0-FA728A274115', line: 0, state: 2, - executedInCurrentKernel: true, + executeKernelId: 'true', data: { source: `x = [1,2,3,4,5]\ny = [21,9,15,17,4]\nprint('This is some irrelevant code')`, cell_type: 'code', @@ -45,7 +45,7 @@ suite('DataScience code gathering unit tests', () => { file: '2DB9B899-6519-4E1B-88B0-FA728A274115', line: 0, state: 2, - executedInCurrentKernel: true, + executeKernelId: 'true', data: { source: `p=figure(title='demo',x_axis_label='x',y_axis_label='y')`, cell_type: 'code', @@ -59,7 +59,7 @@ suite('DataScience code gathering unit tests', () => { file: '2DB9B899-6519-4E1B-88B0-FA728A274115', line: 0, state: 2, - executedInCurrentKernel: true, + executeKernelId: 'true', data: { source: 'p.line(x,y,line_width=2)', cell_type: 'code', @@ -73,7 +73,7 @@ suite('DataScience code gathering unit tests', () => { file: '2DB9B899-6519-4E1B-88B0-FA728A274115', line: 0, state: 2, - executedInCurrentKernel: true, + executeKernelId: 'true', data: { source: 'show(p)', cell_type: 'code', diff --git a/src/test/datascience/jupyterVariables.unit.test.ts b/src/test/datascience/jupyterVariables.unit.test.ts index f6789568a8d8..561135c4c38d 100644 --- a/src/test/datascience/jupyterVariables.unit.test.ts +++ b/src/test/datascience/jupyterVariables.unit.test.ts @@ -36,7 +36,7 @@ suite('JupyterVariables', () => { file: '', line: 0, state: CellState.finished, - executedInCurrentKernel: false + executeKernelId: undefined }; } diff --git a/src/test/datascience/mockJupyterSession.ts b/src/test/datascience/mockJupyterSession.ts index 63cfaa1f5340..0cc9757963ee 100644 --- a/src/test/datascience/mockJupyterSession.ts +++ b/src/test/datascience/mockJupyterSession.ts @@ -28,6 +28,10 @@ export class MockJupyterSession implements IJupyterSession { this.timedelay = timedelay; } + public getKernelId(): string | undefined { + return 'fake_session_kernel_id'; + } + public get onRestarted(): Event { return this.restartedEvent.event; } @@ -102,6 +106,10 @@ export class MockJupyterSession implements IJupyterSession { }; } + public id(): string | undefined { + return 'fake_kernel_id'; + } + public dispose(): Promise { return sleep(10); } From 7b13a51e97c74ba104fa3625da76bd696d0b4603 Mon Sep 17 00:00:00 2001 From: Jim Griesmer Date: Tue, 29 Oct 2019 00:36:41 -0700 Subject: [PATCH 12/42] Add ConnectedToNotebook --- src/client/datascience/constants.ts | 3 ++- .../interactive-common/interactiveBase.ts | 19 ++++++++++++++++++- .../interactiveWindowTypes.ts | 2 ++ .../interactive-ipynb/nativeEditor.ts | 5 +++-- src/client/telemetry/index.ts | 5 +++-- .../interactive-common/mainStateController.ts | 11 +++++++++-- .../native-editor/nativeCell.tsx | 2 ++ 7 files changed, 39 insertions(+), 8 deletions(-) diff --git a/src/client/datascience/constants.ts b/src/client/datascience/constants.ts index c8638639dbb2..6266e196dfee 100644 --- a/src/client/datascience/constants.ts +++ b/src/client/datascience/constants.ts @@ -202,7 +202,8 @@ export enum Telemetry { NotebookRunCount = 'DATASCIENCE.NATIVE.NOTEBOOK_RUN_COUNT', NotebookOpenCount = 'DATASCIENCE.NATIVE.NOTEBOOK_OPEN_COUNT', NotebookOpenTime = 'DS_INTERNAL.NATIVE.NOTEBOOK_OPEN_TIME', - SessionIdleTimeout = 'DATASCIENCE.JUPYTER_IDLE_TIMEOUT' + SessionIdleTimeout = 'DATASCIENCE.JUPYTER_IDLE_TIMEOUT', + ConnectedToNotebook = 'DATASCIENCE.JUPYTER.CONNECTED' } export enum NativeKeyboardCommandTelemetry { diff --git a/src/client/datascience/interactive-common/interactiveBase.ts b/src/client/datascience/interactive-common/interactiveBase.ts index b4ba0896c6b5..52f921262a8b 100644 --- a/src/client/datascience/interactive-common/interactiveBase.ts +++ b/src/client/datascience/interactive-common/interactiveBase.ts @@ -181,6 +181,10 @@ export abstract class InteractiveBase extends WebViewHost { if (this.notebook && !this.restartingKernel) { @@ -688,7 +699,6 @@ export abstract class InteractiveBase extends WebViewHostthis.getNotebook()).getKernelId() : undefined; // Make this error our cell output this.sendCellsToWebView([ { @@ -383,12 +384,12 @@ export class NativeEditor extends InteractiveBase implements INotebookEditor { file: Identifiers.EmptyFileName, line: 0, state: CellState.error, - executeKernelId: this.getNotebook() ? (this.getNotebook()).getKernelId() : undefined + executeKernelId: kernelId } ]); // Tell the other side we restarted the kernel. This will stop all executions - this.postMessage(InteractiveWindowMessages.RestartKernel).ignoreErrors(); + this.postMessage(InteractiveWindowMessages.RestartKernel, kernelId).ignoreErrors(); // Handle an error await this.errorHandler.handleError(exc); diff --git a/src/client/telemetry/index.ts b/src/client/telemetry/index.ts index 702740ff9a1c..5a026fea29d0 100644 --- a/src/client/telemetry/index.ts +++ b/src/client/telemetry/index.ts @@ -1362,6 +1362,7 @@ export interface IEventNamePropertyMapping { [Telemetry.CodeLensAverageAcquisitionTime]: never | undefined; [Telemetry.CollapseAll]: never | undefined; [Telemetry.ConnectFailedJupyter]: never | undefined; + [Telemetry.ConnectedToNotebook]: never | undefined; [Telemetry.ConnectLocalJupyter]: never | undefined; [Telemetry.ConnectRemoteJupyter]: never | undefined; [Telemetry.ConnectRemoteFailedJupyter]: never | undefined; @@ -1425,9 +1426,9 @@ export interface IEventNamePropertyMapping { [Telemetry.RunFileInteractive]: never | undefined; [Telemetry.RunFromLine]: never | undefined; [Telemetry.ScrolledToCell]: never | undefined; - [Telemetry.CellCount]: { count: number} ; + [Telemetry.CellCount]: { count: number }; [Telemetry.Save]: never | undefined; - [Telemetry.AutoSaveEnabled]: {enabled: boolean}; + [Telemetry.AutoSaveEnabled]: { enabled: boolean }; [Telemetry.SelfCertsMessageClose]: never | undefined; [Telemetry.SelfCertsMessageEnabled]: never | undefined; [Telemetry.SelectJupyterURI]: never | undefined; diff --git a/src/datascience-ui/interactive-common/mainStateController.ts b/src/datascience-ui/interactive-common/mainStateController.ts index d97c5b267073..1bcd91f8f8de 100644 --- a/src/datascience-ui/interactive-common/mainStateController.ts +++ b/src/datascience-ui/interactive-common/mainStateController.ts @@ -222,6 +222,10 @@ export class MainStateController implements IMessageHandler { this.handleTmLanguageResponse(payload); break; + case InteractiveWindowMessages.ConnectedToNotebook: + this.handleConnected(payload); + break; + case InteractiveWindowMessages.RestartKernel: // Go through all vms that are currently executing and mark them as finished this.handleRestarted(payload); @@ -966,6 +970,10 @@ export class MainStateController implements IMessageHandler { } } + private handleConnected(payload: string | undefined) { + this.setState({ kernelId: payload }); + } + private handleRestarted(payload: string | undefined) { this.suspendUpdates(); @@ -981,8 +989,7 @@ export class MainStateController implements IMessageHandler { }); this.setState({ cellVMs: newVMs }); } - this.setState({ currentExecutionCount: 0 }); - this.setState({ kernelId: payload }); + this.setState({ currentExecutionCount: 0, kernelId: payload }); this.resumeUpdates(); // Update our variables diff --git a/src/datascience-ui/native-editor/nativeCell.tsx b/src/datascience-ui/native-editor/nativeCell.tsx index 0e0b265fb9f8..27c9e3d71ce8 100644 --- a/src/datascience-ui/native-editor/nativeCell.tsx +++ b/src/datascience-ui/native-editor/nativeCell.tsx @@ -580,6 +580,8 @@ export class NativeCell extends React.Component { this.props.stateController.getState().kernelId === null || this.props.cellVM.cell.executeKernelId !== this.props.stateController.getState().kernelId || this.props.cellVM.cell.data.cell_type !== 'code' || + (this.props.cellVM.cell.state !== CellState.finished && + this.props.cellVM.cell.state !== CellState.error) || getSettings().enableGather === false; }; const canRunAbove = this.props.stateController.canRunAbove(cellId); From 839a75d8dc38fc78944ae741552bbd069d228b82 Mon Sep 17 00:00:00 2001 From: Jim Griesmer Date: Tue, 29 Oct 2019 12:30:30 -0700 Subject: [PATCH 13/42] Simplify logic to make use of cellVM --- src/client/datascience/cellFactory.ts | 4 +-- .../datascience/gather/gatherListener.ts | 1 - .../interactive-common/interactiveBase.ts | 1 - .../interactive-ipynb/nativeEditor.ts | 9 ------ .../datascience/jupyter/jupyterExporter.ts | 1 - .../datascience/jupyter/jupyterImporter.ts | 1 - .../datascience/jupyter/jupyterNotebook.ts | 3 +- src/client/datascience/types.ts | 1 - .../interactive-common/mainState.ts | 6 ++-- .../interactive-common/mainStateController.ts | 28 +++++++++++++------ .../native-editor/nativeCell.tsx | 9 ++---- .../cellhashprovider.unit.test.ts | 1 - .../datascience/gather/gather.unit.test.ts | 5 ---- .../datascience/jupyterVariables.unit.test.ts | 1 - 14 files changed, 26 insertions(+), 45 deletions(-) diff --git a/src/client/datascience/cellFactory.ts b/src/client/datascience/cellFactory.ts index 38622c85303f..9aec18e86ac0 100644 --- a/src/client/datascience/cellFactory.ts +++ b/src/client/datascience/cellFactory.ts @@ -43,8 +43,7 @@ function generateCodeCell(code: string[], file: string, line: number, id: string id: id, file: file, line: line, - state: CellState.init, - executeKernelId: undefined + state: CellState.init }; } @@ -55,7 +54,6 @@ function generateMarkdownCell(code: string[], file: string, line: number, id: st file: file, line: line, state: CellState.finished, - executeKernelId: undefined, data: { cell_type: 'markdown', source: generateMarkdownFromCodeLines(code), diff --git a/src/client/datascience/gather/gatherListener.ts b/src/client/datascience/gather/gatherListener.ts index 09021acb3505..c79084267bc1 100644 --- a/src/client/datascience/gather/gatherListener.ts +++ b/src/client/datascience/gather/gatherListener.ts @@ -62,7 +62,6 @@ export class GatherListener implements IInteractiveWindowListener { file: '', line: 0, state: 0, - executeKernelId: undefined, data: { cell_type: 'markdown', source: localize.DataScience.gatheredNotebookDescriptionInMarkdown(), diff --git a/src/client/datascience/interactive-common/interactiveBase.ts b/src/client/datascience/interactive-common/interactiveBase.ts index 52f921262a8b..5e3064a117af 100644 --- a/src/client/datascience/interactive-common/interactiveBase.ts +++ b/src/client/datascience/interactive-common/interactiveBase.ts @@ -572,7 +572,6 @@ export abstract class InteractiveBase extends WebViewHost { const index = this.visibleCells.findIndex(v => v.id === c.id); this.visibleCells[index] = c; - if (c.state === CellState.executing) { - const n = this.getNotebook(); - if (n !== undefined) { - c.executeKernelId = n.getKernelId(); - } - } }); // Indicate dirty @@ -524,7 +517,6 @@ export class NativeEditor extends InteractiveBase implements INotebookEditor { file: Identifiers.EmptyFileName, line: 0, state: CellState.finished, - executeKernelId: undefined, data: c }; }), forceDirty); @@ -539,7 +531,6 @@ export class NativeEditor extends InteractiveBase implements INotebookEditor { line: 0, file: Identifiers.EmptyFileName, state: CellState.finished, - executeKernelId: undefined, data: { cell_type: 'code', outputs: [], diff --git a/src/client/datascience/jupyter/jupyterExporter.ts b/src/client/datascience/jupyter/jupyterExporter.ts index 904daf2aff2d..0a5a1f6bec2e 100644 --- a/src/client/datascience/jupyter/jupyterExporter.ts +++ b/src/client/datascience/jupyter/jupyterExporter.ts @@ -90,7 +90,6 @@ export class JupyterExporter implements INotebookExporter { file: Identifiers.EmptyFileName, line: 0, state: CellState.finished, - executeKernelId: undefined }; return [cell, ...cells]; diff --git a/src/client/datascience/jupyter/jupyterImporter.ts b/src/client/datascience/jupyter/jupyterImporter.ts index 4455c1b41b95..b0b644207634 100644 --- a/src/client/datascience/jupyter/jupyterImporter.ts +++ b/src/client/datascience/jupyter/jupyterImporter.ts @@ -103,7 +103,6 @@ export class JupyterImporter implements INotebookImporter { file: Identifiers.EmptyFileName, line: 0, state: CellState.finished, - executeKernelId: undefined, data: c, type: 'preview' }; diff --git a/src/client/datascience/jupyter/jupyterNotebook.ts b/src/client/datascience/jupyter/jupyterNotebook.ts index 1bc33056f329..e0d37c159e95 100644 --- a/src/client/datascience/jupyter/jupyterNotebook.ts +++ b/src/client/datascience/jupyter/jupyterNotebook.ts @@ -305,8 +305,7 @@ export class JupyterNotebookBase implements INotebook { id: uuid(), file: '', line: 0, - state: CellState.finished, - executeKernelId: this.getKernelId() + state: CellState.finished }; } diff --git a/src/client/datascience/types.ts b/src/client/datascience/types.ts index 1c5c46eea66f..70622c804acd 100644 --- a/src/client/datascience/types.ts +++ b/src/client/datascience/types.ts @@ -354,7 +354,6 @@ export interface ICell { file: string; line: number; state: CellState; - executeKernelId: string | undefined; data: nbformat.ICodeCell | nbformat.IRawCell | nbformat.IMarkdownCell | IMessageCell; extraLines?: number[]; } diff --git a/src/datascience-ui/interactive-common/mainState.ts b/src/datascience-ui/interactive-common/mainState.ts index 549eed955fbe..9032c9f1936d 100644 --- a/src/datascience-ui/interactive-common/mainState.ts +++ b/src/datascience-ui/interactive-common/mainState.ts @@ -28,6 +28,7 @@ export interface ICellViewModel { useQuickEdit?: boolean; selected: boolean; focused: boolean; + hasBeenRun?: boolean; inputBlockToggled(id: string): void; } @@ -137,7 +138,6 @@ export function createEmptyCell(id: string | undefined, executionCount: number | file: Identifiers.EmptyFileName, line: 0, state: CellState.finished, - executeKernelId: undefined }; } @@ -194,7 +194,8 @@ export function createCellVM(inputCell: ICell, settings: IDataScienceSettings | inputBlockCollapseNeeded: (inputLinesCount > 1), inputBlockToggled: inputBlockToggled, selected: false, - focused: false + focused: false, + hasBeenRun: false }; } @@ -219,7 +220,6 @@ export function generateCells(filePath: string, repetitions: number): ICell[] { file: path.join(filePath, 'foo.py').toLowerCase(), line: 1, state: key === cellData.length - 1 ? CellState.executing : CellState.finished, - executeKernelId: undefined, type: key === 3 ? 'preview' : 'execute', data: data }; diff --git a/src/datascience-ui/interactive-common/mainStateController.ts b/src/datascience-ui/interactive-common/mainStateController.ts index 1bcd91f8f8de..a3ccfc5d0332 100644 --- a/src/datascience-ui/interactive-common/mainStateController.ts +++ b/src/datascience-ui/interactive-common/mainStateController.ts @@ -977,19 +977,27 @@ export class MainStateController implements IMessageHandler { private handleRestarted(payload: string | undefined) { this.suspendUpdates(); - // When we restart, make sure to turn off all executing cells. They aren't executing anymore - const executingCells = this.pendingState.cellVMs + const newVMs = [...this.pendingState.cellVMs]; + + // When we restart, reset all code cells to indicate they haven't been run in the new kernel. + // Also make sure to turn off all executing cells as they aren't executing anymore. + const executableCells = newVMs .map((cvm, i) => { return { cvm, i }; }) - .filter(s => s.cvm.cell.state !== CellState.error && s.cvm.cell.state !== CellState.finished); + .filter(s => s.cvm.cell.data.cell_type === 'code'); - if (executingCells && executingCells.length) { - const newVMs = [...this.pendingState.cellVMs]; - executingCells.forEach(s => { - newVMs[s.i] = immutable.updateIn(s.cvm, ['cell', 'state'], () => CellState.finished); + if (executableCells && executableCells.length) { + executableCells.forEach(s => { + if (newVMs[s.i].hasBeenRun && newVMs[s.i].hasBeenRun === true) { + newVMs[s.i] = immutable.updateIn(s.cvm, ['hasBeenRun'], () => false); + } + + if (newVMs[s.i].cell.state !== CellState.error && newVMs[s.i].cell.state !== CellState.finished) { + newVMs[s.i] = immutable.updateIn(s.cvm, ['cell', 'state'], () => CellState.finished); + } }); - this.setState({ cellVMs: newVMs }); } - this.setState({ currentExecutionCount: 0, kernelId: payload }); + + this.setState({ cellVMs: newVMs, currentExecutionCount: 0, kernelId: payload }); this.resumeUpdates(); // Update our variables @@ -1160,6 +1168,8 @@ export class MainStateController implements IMessageHandler { this.refreshVariables(newExecutionCount); } + this.pendingState.cellVMs[index].hasBeenRun = true; + // Have to make a copy of the cell VM array or // we won't actually update. const newVMs = [...this.pendingState.cellVMs]; diff --git a/src/datascience-ui/native-editor/nativeCell.tsx b/src/datascience-ui/native-editor/nativeCell.tsx index 27c9e3d71ce8..d2e09ac57387 100644 --- a/src/datascience-ui/native-editor/nativeCell.tsx +++ b/src/datascience-ui/native-editor/nativeCell.tsx @@ -575,13 +575,8 @@ export class NativeCell extends React.Component { }; const gatherDisabled = () => { return this.props.cellVM.cell.data.execution_count === null || - this.props.cellVM.cell.executeKernelId === null || - this.props.cellVM.cell.executeKernelId === undefined || - this.props.stateController.getState().kernelId === null || - this.props.cellVM.cell.executeKernelId !== this.props.stateController.getState().kernelId || - this.props.cellVM.cell.data.cell_type !== 'code' || - (this.props.cellVM.cell.state !== CellState.finished && - this.props.cellVM.cell.state !== CellState.error) || + this.props.cellVM.hasBeenRun === null || + this.props.cellVM.hasBeenRun === false || getSettings().enableGather === false; }; const canRunAbove = this.props.stateController.canRunAbove(cellId); diff --git a/src/test/datascience/editor-integration/cellhashprovider.unit.test.ts b/src/test/datascience/editor-integration/cellhashprovider.unit.test.ts index 646ecedc8269..560aa5d9428f 100644 --- a/src/test/datascience/editor-integration/cellhashprovider.unit.test.ts +++ b/src/test/datascience/editor-integration/cellhashprovider.unit.test.ts @@ -69,7 +69,6 @@ suite('CellHashProvider Unit Tests', () => { }, id: '1', state: CellState.init, - executeKernelId: undefined }; return hashProvider.preExecute(cell, false); } diff --git a/src/test/datascience/gather/gather.unit.test.ts b/src/test/datascience/gather/gather.unit.test.ts index da9754d5c1c9..81c23eb66fa2 100644 --- a/src/test/datascience/gather/gather.unit.test.ts +++ b/src/test/datascience/gather/gather.unit.test.ts @@ -17,7 +17,6 @@ suite('DataScience code gathering unit tests', () => { file: '2DB9B899-6519-4E1B-88B0-FA728A274115', line: 0, state: 2, - executeKernelId: 'true', data: { source: `from bokeh.plotting import show, figure, output_notebook\noutput_notebook()`, cell_type: 'code', @@ -31,7 +30,6 @@ suite('DataScience code gathering unit tests', () => { file: '2DB9B899-6519-4E1B-88B0-FA728A274115', line: 0, state: 2, - executeKernelId: 'true', data: { source: `x = [1,2,3,4,5]\ny = [21,9,15,17,4]\nprint('This is some irrelevant code')`, cell_type: 'code', @@ -45,7 +43,6 @@ suite('DataScience code gathering unit tests', () => { file: '2DB9B899-6519-4E1B-88B0-FA728A274115', line: 0, state: 2, - executeKernelId: 'true', data: { source: `p=figure(title='demo',x_axis_label='x',y_axis_label='y')`, cell_type: 'code', @@ -59,7 +56,6 @@ suite('DataScience code gathering unit tests', () => { file: '2DB9B899-6519-4E1B-88B0-FA728A274115', line: 0, state: 2, - executeKernelId: 'true', data: { source: 'p.line(x,y,line_width=2)', cell_type: 'code', @@ -73,7 +69,6 @@ suite('DataScience code gathering unit tests', () => { file: '2DB9B899-6519-4E1B-88B0-FA728A274115', line: 0, state: 2, - executeKernelId: 'true', data: { source: 'show(p)', cell_type: 'code', diff --git a/src/test/datascience/jupyterVariables.unit.test.ts b/src/test/datascience/jupyterVariables.unit.test.ts index 561135c4c38d..abf097e9b58e 100644 --- a/src/test/datascience/jupyterVariables.unit.test.ts +++ b/src/test/datascience/jupyterVariables.unit.test.ts @@ -36,7 +36,6 @@ suite('JupyterVariables', () => { file: '', line: 0, state: CellState.finished, - executeKernelId: undefined }; } From 0e1b8fce4326aeaf4e7971540708159331c5e8be Mon Sep 17 00:00:00 2001 From: Jim Griesmer Date: Wed, 30 Oct 2019 00:48:10 -0700 Subject: [PATCH 14/42] Attempt to fix multi-nb gather (failed) --- .../datascience/gather/gatherListener.ts | 50 ++----------------- .../interactive-common/interactiveBase.ts | 4 +- .../interactiveWindowTypes.ts | 8 +-- .../interactive-ipynb/nativeEditor.ts | 49 +++++++++++++++++- .../interactive-ipynb/nativeEditorProvider.ts | 8 ++- .../datascience/jupyter/jupyterNotebook.ts | 18 +++++++ .../jupyter/liveshare/guestJupyterNotebook.ts | 14 +++++- src/client/datascience/serviceRegistry.ts | 2 +- src/client/datascience/types.ts | 5 +- .../interactive-common/mainStateController.ts | 2 +- src/test/datascience/execution.unit.test.ts | 13 +++++ 11 files changed, 114 insertions(+), 59 deletions(-) diff --git a/src/client/datascience/gather/gatherListener.ts b/src/client/datascience/gather/gatherListener.ts index c79084267bc1..a05a7c356e8e 100644 --- a/src/client/datascience/gather/gatherListener.ts +++ b/src/client/datascience/gather/gatherListener.ts @@ -1,12 +1,8 @@ import { inject, injectable } from 'inversify'; -import * as uuid from 'uuid/v4'; import { Event, EventEmitter } from 'vscode'; -import { IApplicationShell } from '../../common/application/types'; -import * as localize from '../../common/utils/localize'; import { noop } from '../../common/utils/misc'; -import { generateCellsFromString } from '../cellFactory'; import { InteractiveWindowMessages } from '../interactive-common/interactiveWindowTypes'; -import { ICell, IGatherExecution, IInteractiveWindowListener, INotebookEditorProvider, INotebookExporter } from '../types'; +import { IGatherExecution, IInteractiveWindowListener, INotebook } from '../types'; @injectable() export class GatherListener implements IInteractiveWindowListener { @@ -14,9 +10,6 @@ export class GatherListener implements IInteractiveWindowListener { private postEmitter: EventEmitter<{ message: string; payload: any }> = new EventEmitter<{ message: string; payload: any }>(); constructor( - @inject(INotebookExporter) private jupyterExporter: INotebookExporter, - @inject(INotebookEditorProvider) private ipynbProvider: INotebookEditorProvider, - @inject(IApplicationShell) private applicationShell: IApplicationShell, @inject(IGatherExecution) private gatherExecution: IGatherExecution) { } public dispose() { @@ -31,50 +24,15 @@ export class GatherListener implements IInteractiveWindowListener { // tslint:disable-next-line: no-any public onMessage(message: string, payload?: any): void { switch (message) { - case InteractiveWindowMessages.GatherCode: + case InteractiveWindowMessages.ConnectedToNotebook: if (payload) { - const cell = payload as ICell; - this.gatherCode(cell); + const nb = payload as INotebook; + void nb.setGatherHandler(this.gatherExecution); } break; - case InteractiveWindowMessages.RestartKernel: - this.gatherExecution.resetLog(); - break; - default: break; } } - - public gatherCode(payload: ICell) { - this.gatherCodeInternal(payload).catch(err => { - this.applicationShell.showErrorMessage(err); - }); - } - - private gatherCodeInternal = async (cell: ICell) => { - const slicedProgram = this.gatherExecution.gatherCode(cell); - const newuri = await this.ipynbProvider.getNextNewNotebookUri(); - - let cells: ICell[] = [{ - id: uuid(), - file: '', - line: 0, - state: 0, - data: { - cell_type: 'markdown', - source: localize.DataScience.gatheredNotebookDescriptionInMarkdown(), - metadata: {} - } - }]; - - // Create new notebook with the returned program and open it. - cells = cells.concat(generateCellsFromString(slicedProgram)); - - const notebook = await this.jupyterExporter.translateToNotebook(cells); - const contents = JSON.stringify(notebook); - - await this.ipynbProvider.open(newuri, contents); - } } diff --git a/src/client/datascience/interactive-common/interactiveBase.ts b/src/client/datascience/interactive-common/interactiveBase.ts index 5e3064a117af..498706a3e212 100644 --- a/src/client/datascience/interactive-common/interactiveBase.ts +++ b/src/client/datascience/interactive-common/interactiveBase.ts @@ -327,7 +327,7 @@ export abstract class InteractiveBase extends WebViewHost { + this.applicationShell.showErrorMessage(err); + }); + } + + private gatherCodeInternal = async (cell: ICell) => { + const nb = this.getNotebook(); + if (nb) { + const slicedProgram = nb.gatherCode(cell); + if (slicedProgram) { + let cells: ICell[] = [{ + id: uuid(), + file: '', + line: 0, + state: 0, + data: { + cell_type: 'markdown', + source: localize.DataScience.gatheredNotebookDescriptionInMarkdown(), + metadata: {} + } + }]; + + // Create new notebook with the returned program and open it. + cells = cells.concat(generateCellsFromString(slicedProgram)); + + const notebook = await this.jupyterExporter.translateToNotebook(cells); + const contents = JSON.stringify(notebook); + await this.ipynbProvider.createNew(contents); + } + } + } + private async askForSave(): Promise { const message1 = localize.DataScience.dirtyNotebookMessage1().format(`${path.basename(this.file.fsPath)}`); const message2 = localize.DataScience.dirtyNotebookMessage2(); diff --git a/src/client/datascience/interactive-ipynb/nativeEditorProvider.ts b/src/client/datascience/interactive-ipynb/nativeEditorProvider.ts index 06319e9078b9..5023991edc5b 100644 --- a/src/client/datascience/interactive-ipynb/nativeEditorProvider.ts +++ b/src/client/datascience/interactive-ipynb/nativeEditorProvider.ts @@ -105,11 +105,15 @@ export class NativeEditorProvider implements INotebookEditorProvider, IAsyncDisp } @captureTelemetry(Telemetry.CreateNewNotebook, undefined, false) - public async createNew(): Promise { + public async createNew(contents?: string): Promise { // Create a new URI for the dummy file using our root workspace path const uri = await this.getNextNewNotebookUri(); this.notebookCount += 1; - return this.open(uri, ''); + if (contents) { + return this.open(uri, contents); + } else { + return this.open(uri, ''); + } } public async getNotebookOptions(): Promise { diff --git a/src/client/datascience/jupyter/jupyterNotebook.ts b/src/client/datascience/jupyter/jupyterNotebook.ts index e0d37c159e95..9751d80b6ca7 100644 --- a/src/client/datascience/jupyter/jupyterNotebook.ts +++ b/src/client/datascience/jupyter/jupyterNotebook.ts @@ -28,6 +28,7 @@ import { CodeSnippits, Identifiers, Telemetry } from '../constants'; import { CellState, ICell, + IGatherExecution, IJupyterSession, INotebook, INotebookCompletion, @@ -138,6 +139,7 @@ export class JupyterNotebookBase implements INotebook { private sessionStartTime: number; private pendingCellSubscriptions: CellSubscriber[] = []; private ranInitialSetup = false; + private gather: IGatherExecution | undefined; private _resource: Uri; private _disposed: boolean = false; private _workingDirectory: string | undefined; @@ -255,6 +257,22 @@ export class JupyterNotebookBase implements INotebook { return this.updateWorkingDirectory(file); } + public setGatherHandler(g: IGatherExecution) { + this.gather = g; + } + + public gatherCode(c: ICell): string | undefined { + if (this.gather) { + return this.gather.gatherCode(c); + } + } + + public resetLog() { + if (this.gather) { + this.gather.resetLog(); + } + } + public executeObservable(code: string, file: string, line: number, id: string, silent: boolean = false): Observable { // Create an observable and wrap the result so we can time it. const stopWatch = new StopWatch(); diff --git a/src/client/datascience/jupyter/liveshare/guestJupyterNotebook.ts b/src/client/datascience/jupyter/liveshare/guestJupyterNotebook.ts index f7fe9e588881..27505ad382c2 100644 --- a/src/client/datascience/jupyter/liveshare/guestJupyterNotebook.ts +++ b/src/client/datascience/jupyter/liveshare/guestJupyterNotebook.ts @@ -14,7 +14,7 @@ import { createDeferred } from '../../../common/utils/async'; import * as localize from '../../../common/utils/localize'; import { noop } from '../../../common/utils/misc'; import { LiveShare, LiveShareCommands } from '../../constants'; -import { ICell, INotebook, INotebookCompletion, INotebookServer, InterruptResult } from '../../types'; +import { ICell, IGatherExecution, INotebook, INotebookCompletion, INotebookServer, InterruptResult } from '../../types'; import { LiveShareParticipantDefault, LiveShareParticipantGuest } from './liveShareParticipantMixin'; import { ResponseQueue } from './responseQueue'; import { IExecuteObservableResponse, ILiveShareParticipant, IServerResponse } from './types'; @@ -59,6 +59,14 @@ export class GuestJupyterNotebook return 'NA'; } + public gatherCode(_c: ICell): string | undefined { + return undefined; + } + + public resetLog(): void { + return; + } + public clear(_id: string): void { // We don't do anything as we don't cache results in this class. noop(); @@ -96,6 +104,10 @@ export class GuestJupyterNotebook return Promise.resolve(); } + public setGatherHandler(_gather: IGatherExecution): void { + noop(); + } + public async setMatplotLibStyle(_useDark: boolean): Promise { // Guest can't change the style. Maybe output a warning here? } diff --git a/src/client/datascience/serviceRegistry.ts b/src/client/datascience/serviceRegistry.ts index 435b6285d99b..00c56d21e48a 100644 --- a/src/client/datascience/serviceRegistry.ts +++ b/src/client/datascience/serviceRegistry.ts @@ -131,7 +131,7 @@ export function registerTypes(serviceManager: IServiceManager) { serviceManager.add(IDataScienceErrorHandler, wrapType(DataScienceErrorHandler)); serviceManager.addSingleton(ICodeLensFactory, wrapType(CodeLensFactory)); serviceManager.addSingleton(ICellHashProvider, wrapType(CellHashProvider)); - serviceManager.addSingleton(IGatherExecution, wrapType(GatherExecution)); + serviceManager.add(IGatherExecution, wrapType(GatherExecution)); serviceManager.addBinding(ICellHashProvider, IInteractiveWindowListener); serviceManager.addBinding(ICellHashProvider, INotebookExecutionLogger); serviceManager.addBinding(IJupyterDebugger, ICellHashListener); diff --git a/src/client/datascience/types.ts b/src/client/datascience/types.ts index 70622c804acd..facb6bafab3f 100644 --- a/src/client/datascience/types.ts +++ b/src/client/datascience/types.ts @@ -99,6 +99,9 @@ export interface INotebook extends IAsyncDisposable { getKernelId(): string | undefined; getSysInfo(): Promise; setMatplotLibStyle(useDark: boolean): Promise; + setGatherHandler(gather: IGatherExecution): void; + gatherCode(cell: ICell): string | undefined; + resetLog(): void; } export interface INotebookServerOptions { @@ -244,7 +247,7 @@ export interface INotebookEditorProvider { readonly editors: INotebookEditor[]; open(file: Uri, contents: string): Promise; show(file: Uri): Promise; - createNew(): Promise; + createNew(contents?: string): Promise; getNotebookOptions(): Promise; getNextNewNotebookUri(): Promise; } diff --git a/src/datascience-ui/interactive-common/mainStateController.ts b/src/datascience-ui/interactive-common/mainStateController.ts index a3ccfc5d0332..6c2c58604324 100644 --- a/src/datascience-ui/interactive-common/mainStateController.ts +++ b/src/datascience-ui/interactive-common/mainStateController.ts @@ -435,7 +435,7 @@ export class MainStateController implements IMessageHandler { public gatherCell = (cellVM: ICellViewModel | undefined) => { if (cellVM) { - this.sendMessage(InteractiveWindowMessages.GatherCode, cellVM.cell); + this.sendMessage(InteractiveWindowMessages.GatherCodeRequest, cellVM.cell); } } diff --git a/src/test/datascience/execution.unit.test.ts b/src/test/datascience/execution.unit.test.ts index 38defebbabbd..6756c781eac4 100644 --- a/src/test/datascience/execution.unit.test.ts +++ b/src/test/datascience/execution.unit.test.ts @@ -41,6 +41,7 @@ import { JupyterExecutionFactory } from '../../client/datascience/jupyter/jupyte import { ICell, IConnection, + IGatherExecution, IJupyterKernelSpec, INotebook, INotebookCompletion, @@ -103,6 +104,10 @@ class MockJupyterNotebook implements INotebook { noop(); } + public setGatherHandler(_gather: IGatherExecution): void { + noop(); + } + public getSysInfo(): Promise { return Promise.resolve(undefined); } @@ -118,6 +123,14 @@ class MockJupyterNotebook implements INotebook { public async dispose(): Promise { return Promise.resolve(); } + + public gatherCode(_c: ICell): string | undefined { + return undefined; + } + + public resetLog(): void { + return; + } } // tslint:disable:no-any no-http-string no-multiline-string max-func-body-length From a0094e51d3f198c30efb99c7f48bb44642b4100b Mon Sep 17 00:00:00 2001 From: Jim Griesmer Date: Wed, 30 Oct 2019 22:36:17 -0700 Subject: [PATCH 15/42] Support separate notebook gathers --- src/client/datascience/gather/gatherListener.ts | 6 +++--- .../datascience/interactive-common/interactiveBase.ts | 4 ++-- .../datascience/interactive-ipynb/nativeEditor.ts | 10 ---------- src/client/datascience/jupyter/jupyterNotebook.ts | 6 ++++-- .../jupyter/liveshare/guestJupyterNotebook.ts | 4 ++-- src/client/datascience/serviceRegistry.ts | 1 - src/client/datascience/types.ts | 4 ++-- src/test/datascience/execution.unit.test.ts | 4 ++-- 8 files changed, 15 insertions(+), 24 deletions(-) diff --git a/src/client/datascience/gather/gatherListener.ts b/src/client/datascience/gather/gatherListener.ts index a05a7c356e8e..ce7fbe8cf67f 100644 --- a/src/client/datascience/gather/gatherListener.ts +++ b/src/client/datascience/gather/gatherListener.ts @@ -25,9 +25,9 @@ export class GatherListener implements IInteractiveWindowListener { public onMessage(message: string, payload?: any): void { switch (message) { case InteractiveWindowMessages.ConnectedToNotebook: - if (payload) { - const nb = payload as INotebook; - void nb.setGatherHandler(this.gatherExecution); + const nb = payload as INotebook; + if (nb) { + void nb.addGatherSupport(this.gatherExecution); } break; diff --git a/src/client/datascience/interactive-common/interactiveBase.ts b/src/client/datascience/interactive-common/interactiveBase.ts index 498706a3e212..13ba785fe29b 100644 --- a/src/client/datascience/interactive-common/interactiveBase.ts +++ b/src/client/datascience/interactive-common/interactiveBase.ts @@ -326,8 +326,8 @@ export abstract class InteractiveBase extends WebViewHost(INotebookEditorProvider, wrapType(NativeEditorProvider)); serviceManager.add(INotebookEditor, wrapType(NativeEditor)); serviceManager.addSingleton(IDataScienceCommandListener, wrapType(NativeEditorCommandListener)); - serviceManager.addBinding(IGatherExecution, INotebookExecutionLogger); serviceManager.addBinding(ICodeLensFactory, IInteractiveWindowListener); serviceManager.addSingleton(IDebugLocationTracker, wrapType(DebugLocationTrackerFactory)); } diff --git a/src/client/datascience/types.ts b/src/client/datascience/types.ts index facb6bafab3f..8216ed2306e8 100644 --- a/src/client/datascience/types.ts +++ b/src/client/datascience/types.ts @@ -99,9 +99,9 @@ export interface INotebook extends IAsyncDisposable { getKernelId(): string | undefined; getSysInfo(): Promise; setMatplotLibStyle(useDark: boolean): Promise; - setGatherHandler(gather: IGatherExecution): void; + addGatherSupport(gather: IGatherExecution): void; gatherCode(cell: ICell): string | undefined; - resetLog(): void; + resetGatherLog(): void; } export interface INotebookServerOptions { diff --git a/src/test/datascience/execution.unit.test.ts b/src/test/datascience/execution.unit.test.ts index 6756c781eac4..4da83850f3c8 100644 --- a/src/test/datascience/execution.unit.test.ts +++ b/src/test/datascience/execution.unit.test.ts @@ -104,7 +104,7 @@ class MockJupyterNotebook implements INotebook { noop(); } - public setGatherHandler(_gather: IGatherExecution): void { + public addGatherSupport(_gather: IGatherExecution): void { noop(); } @@ -128,7 +128,7 @@ class MockJupyterNotebook implements INotebook { return undefined; } - public resetLog(): void { + public resetGatherLog(): void { return; } } From b58fe0ed9a1a421bee9c7fbba55d7f3a3a1e7ab8 Mon Sep 17 00:00:00 2001 From: Jim Griesmer Date: Wed, 30 Oct 2019 23:33:39 -0700 Subject: [PATCH 16/42] PR adjustments --- src/client/datascience/gather/gather.ts | 2 +- .../interactive-common/interactiveBase.ts | 3 --- .../interactive-ipynb/nativeEditor.ts | 12 +---------- .../interactive-ipynb/nativeEditorProvider.ts | 21 ++++++++++--------- 4 files changed, 13 insertions(+), 25 deletions(-) diff --git a/src/client/datascience/gather/gather.ts b/src/client/datascience/gather/gather.ts index e2f64e62d5e4..dc23235bbce9 100644 --- a/src/client/datascience/gather/gather.ts +++ b/src/client/datascience/gather/gather.ts @@ -78,7 +78,7 @@ export class GatherExecution implements IGatherExecution, INotebookExecutionLogg */ public gatherCode(vscCell: IVscCell): string { const gatherCell = convertVscToGatherCell(vscCell); - if (gatherCell === undefined) { + if (!gatherCell) { return ''; } diff --git a/src/client/datascience/interactive-common/interactiveBase.ts b/src/client/datascience/interactive-common/interactiveBase.ts index acb140063c1c..9a998cea7542 100644 --- a/src/client/datascience/interactive-common/interactiveBase.ts +++ b/src/client/datascience/interactive-common/interactiveBase.ts @@ -934,9 +934,6 @@ export abstract class InteractiveBase extends WebViewHostthis.getNotebook()).getKernelId() : undefined; // Make this error our cell output this.sendCellsToWebView([ { @@ -399,7 +393,7 @@ export class NativeEditor extends InteractiveBase implements INotebookEditor { ]); // Tell the other side we restarted the kernel. This will stop all executions - this.postMessage(InteractiveWindowMessages.RestartKernel, kernelId).ignoreErrors(); + this.postMessage(InteractiveWindowMessages.RestartKernel).ignoreErrors(); // Handle an error await this.errorHandler.handleError(exc); @@ -927,8 +921,4 @@ export class NativeEditor extends InteractiveBase implements INotebookEditor { sendTelemetryEvent(Telemetry.NotebookOpenTime, this.startupTimer.elapsedTime); } } - - private onRestartKernel() { - this.sendCellsToWebView(this.cells); - } } diff --git a/src/client/datascience/interactive-ipynb/nativeEditorProvider.ts b/src/client/datascience/interactive-ipynb/nativeEditorProvider.ts index ac5cb7557b7a..00d677e4e728 100644 --- a/src/client/datascience/interactive-ipynb/nativeEditorProvider.ts +++ b/src/client/datascience/interactive-ipynb/nativeEditorProvider.ts @@ -18,16 +18,6 @@ import { IDataScienceErrorHandler, INotebookEditor, INotebookEditorProvider, INo @injectable() export class NativeEditorProvider implements INotebookEditorProvider, IAsyncDisposable { - public get activeEditor(): INotebookEditor | undefined { - const active = [...this.activeEditors.entries()].find(e => e[1].active); - if (active) { - return active[1]; - } - } - - public get editors(): INotebookEditor[] { - return [...this.activeEditors.values()]; - } private activeEditors: Map = new Map(); private executedEditors: Set = new Set(); private notebookCount: number = 0; @@ -84,6 +74,17 @@ export class NativeEditorProvider implements INotebookEditorProvider, IAsyncDisp sendTelemetryEvent(Telemetry.NotebookWorkspaceCount, this.notebookCount); } + public get activeEditor(): INotebookEditor | undefined { + const active = [...this.activeEditors.entries()].find(e => e[1].active); + if (active) { + return active[1]; + } + } + + public get editors(): INotebookEditor[] { + return [...this.activeEditors.values()]; + } + public async open(file: Uri, contents: string): Promise { // See if this file is open or not already let editor = this.activeEditors.get(file.fsPath); From c96acd6e794b68425a38fb85f788670a64da5bca Mon Sep 17 00:00:00 2001 From: Jim Griesmer Date: Wed, 30 Oct 2019 23:37:01 -0700 Subject: [PATCH 17/42] More PR updates --- src/client/datascience/jupyter/jupyterNotebook.ts | 4 ---- src/client/datascience/jupyter/jupyterSession.ts | 6 ------ .../datascience/jupyter/liveshare/guestJupyterNotebook.ts | 4 ---- src/client/datascience/types.ts | 2 -- src/test/datascience/execution.unit.test.ts | 4 ---- src/test/datascience/mockJupyterSession.ts | 4 ---- 6 files changed, 24 deletions(-) diff --git a/src/client/datascience/jupyter/jupyterNotebook.ts b/src/client/datascience/jupyter/jupyterNotebook.ts index c0873192b7b4..f7583f59d45a 100644 --- a/src/client/datascience/jupyter/jupyterNotebook.ts +++ b/src/client/datascience/jupyter/jupyterNotebook.ts @@ -293,10 +293,6 @@ export class JupyterNotebookBase implements INotebook { }); } - public getKernelId(): string | undefined { - return this.session.getKernelId(); - } - public async getSysInfo(): Promise { // tslint:disable-next-line:no-multiline-string const versionCells = await this.executeSilently(`import sys\r\nsys.version`); diff --git a/src/client/datascience/jupyter/jupyterSession.ts b/src/client/datascience/jupyter/jupyterSession.ts index e57c43edc617..45d68a528c65 100644 --- a/src/client/datascience/jupyter/jupyterSession.ts +++ b/src/client/datascience/jupyter/jupyterSession.ts @@ -47,12 +47,6 @@ export class JupyterSession implements IJupyterSession { return this.shutdown(); } - public getKernelId(): string | undefined { - if (this.session !== undefined) { - return this.session.id; - } - } - public async shutdown(): Promise { // Destroy the notebook file if not local. Local is cleaned up when we destroy the kernel spec. if (this.notebookFiles.length && this.contentsManager && this.connInfo && !this.connInfo.localLaunch) { diff --git a/src/client/datascience/jupyter/liveshare/guestJupyterNotebook.ts b/src/client/datascience/jupyter/liveshare/guestJupyterNotebook.ts index 559b4c7a1792..6daabb34d188 100644 --- a/src/client/datascience/jupyter/liveshare/guestJupyterNotebook.ts +++ b/src/client/datascience/jupyter/liveshare/guestJupyterNotebook.ts @@ -55,10 +55,6 @@ export class GuestJupyterNotebook return Promise.resolve(); } - public getKernelId(): string | undefined { - return 'NA'; - } - public gatherCode(_c: ICell): string | undefined { return undefined; } diff --git a/src/client/datascience/types.ts b/src/client/datascience/types.ts index bb5d467428d6..7cc572a3f52b 100644 --- a/src/client/datascience/types.ts +++ b/src/client/datascience/types.ts @@ -96,7 +96,6 @@ export interface INotebook extends IAsyncDisposable { waitForIdle(timeoutInMs: number): Promise; interruptKernel(timeoutInMs: number): Promise; setLaunchingFile(file: string): Promise; - getKernelId(): string | undefined; getSysInfo(): Promise; setMatplotLibStyle(useDark: boolean): Promise; addGatherSupport(gather: IGatherExecution): void; @@ -165,7 +164,6 @@ export interface IJupyterPasswordConnect { export const IJupyterSession = Symbol('IJupyterSession'); export interface IJupyterSession extends IAsyncDisposable { onRestarted: Event; - getKernelId(): string | undefined; restart(timeout: number): Promise; interrupt(timeout: number): Promise; waitForIdle(timeout: number): Promise; diff --git a/src/test/datascience/execution.unit.test.ts b/src/test/datascience/execution.unit.test.ts index 4da83850f3c8..f9e7cc5c0707 100644 --- a/src/test/datascience/execution.unit.test.ts +++ b/src/test/datascience/execution.unit.test.ts @@ -112,10 +112,6 @@ class MockJupyterNotebook implements INotebook { return Promise.resolve(undefined); } - public getKernelId(): string | undefined { - return 'fake_kernel_id'; - } - public interruptKernel(_timeout: number): Promise { throw new Error('Method not implemented'); } diff --git a/src/test/datascience/mockJupyterSession.ts b/src/test/datascience/mockJupyterSession.ts index 0cc9757963ee..c08a21869738 100644 --- a/src/test/datascience/mockJupyterSession.ts +++ b/src/test/datascience/mockJupyterSession.ts @@ -28,10 +28,6 @@ export class MockJupyterSession implements IJupyterSession { this.timedelay = timedelay; } - public getKernelId(): string | undefined { - return 'fake_session_kernel_id'; - } - public get onRestarted(): Event { return this.restartedEvent.event; } From 643e17c8ad9c3a51cd3347e87e3167aa9b029723 Mon Sep 17 00:00:00 2001 From: Jim Griesmer Date: Wed, 30 Oct 2019 23:49:10 -0700 Subject: [PATCH 18/42] More PR updates --- .../interactive-common/interactiveBase.ts | 1 + .../interactive-common/interactiveWindowTypes.ts | 2 +- src/datascience-ui/interactive-common/mainState.ts | 3 +-- .../interactive-common/mainStateController.ts | 14 +++----------- .../cellhashprovider.unit.test.ts | 2 +- src/test/datascience/jupyterVariables.unit.test.ts | 2 +- src/test/datascience/mockJupyterSession.ts | 4 ---- 7 files changed, 8 insertions(+), 20 deletions(-) diff --git a/src/client/datascience/interactive-common/interactiveBase.ts b/src/client/datascience/interactive-common/interactiveBase.ts index 9a998cea7542..22dceac410e2 100644 --- a/src/client/datascience/interactive-common/interactiveBase.ts +++ b/src/client/datascience/interactive-common/interactiveBase.ts @@ -698,6 +698,7 @@ export abstract class InteractiveBase extends WebViewHost { execution_count: 1 }, id: '1', - state: CellState.init, + state: CellState.init }; return hashProvider.preExecute(cell, false); } diff --git a/src/test/datascience/jupyterVariables.unit.test.ts b/src/test/datascience/jupyterVariables.unit.test.ts index abf097e9b58e..5efba16b5190 100644 --- a/src/test/datascience/jupyterVariables.unit.test.ts +++ b/src/test/datascience/jupyterVariables.unit.test.ts @@ -35,7 +35,7 @@ suite('JupyterVariables', () => { id: '0', file: '', line: 0, - state: CellState.finished, + state: CellState.finished }; } diff --git a/src/test/datascience/mockJupyterSession.ts b/src/test/datascience/mockJupyterSession.ts index c08a21869738..63cfaa1f5340 100644 --- a/src/test/datascience/mockJupyterSession.ts +++ b/src/test/datascience/mockJupyterSession.ts @@ -102,10 +102,6 @@ export class MockJupyterSession implements IJupyterSession { }; } - public id(): string | undefined { - return 'fake_kernel_id'; - } - public dispose(): Promise { return sleep(10); } From 6a3ca5a57cadd79c2d608f5eb04c78f97ecbcc83 Mon Sep 17 00:00:00 2001 From: Jim Griesmer Date: Thu, 31 Oct 2019 00:00:59 -0700 Subject: [PATCH 19/42] cleanup --- src/client/datascience/interactive-ipynb/nativeEditorProvider.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/client/datascience/interactive-ipynb/nativeEditorProvider.ts b/src/client/datascience/interactive-ipynb/nativeEditorProvider.ts index 00d677e4e728..8a0e0c6b1722 100644 --- a/src/client/datascience/interactive-ipynb/nativeEditorProvider.ts +++ b/src/client/datascience/interactive-ipynb/nativeEditorProvider.ts @@ -73,7 +73,6 @@ export class NativeEditorProvider implements INotebookEditorProvider, IAsyncDisp sendTelemetryEvent(Telemetry.NotebookRunCount, this.executedEditors.size); sendTelemetryEvent(Telemetry.NotebookWorkspaceCount, this.notebookCount); } - public get activeEditor(): INotebookEditor | undefined { const active = [...this.activeEditors.entries()].find(e => e[1].active); if (active) { From 5039dee3006382432a997fd85f59728da07f9a1c Mon Sep 17 00:00:00 2001 From: Jim Griesmer Date: Thu, 31 Oct 2019 00:12:02 -0700 Subject: [PATCH 20/42] Yet another PR update --- .../interactive-ipynb/nativeEditorProvider.ts | 50 +++++++++---------- .../datascience/jupyter/jupyterExporter.ts | 2 +- src/client/datascience/types.ts | 1 - .../datascience/testNativeEditorProvider.ts | 4 -- 4 files changed, 26 insertions(+), 31 deletions(-) diff --git a/src/client/datascience/interactive-ipynb/nativeEditorProvider.ts b/src/client/datascience/interactive-ipynb/nativeEditorProvider.ts index 8a0e0c6b1722..6d089c94eec6 100644 --- a/src/client/datascience/interactive-ipynb/nativeEditorProvider.ts +++ b/src/client/datascience/interactive-ipynb/nativeEditorProvider.ts @@ -135,31 +135,6 @@ export class NativeEditorProvider implements INotebookEditorProvider, IAsyncDisp }; } - public async getNextNewNotebookUri(): Promise { - // Start in the root and look for files starting with untitled - let number = 1; - const dir = this.workspace.rootPath; - if (dir) { - const existing = await this.fileSystem.search(path.join(dir, `${localize.DataScience.untitledNotebookFileName()}-*.ipynb`)); - - // Sort by number - existing.sort(); - - // Add one onto the end of the last one - if (existing.length > 0) { - const match = /(\w+)-(\d+)\.ipynb/.exec(path.basename(existing[existing.length - 1])); - if (match && match.length > 1) { - number = parseInt(match[2], 10); - } - return Uri.file(path.join(dir, `${localize.DataScience.untitledNotebookFileName()}-${number + 1}`)); - } - } - - const result = Uri.file(`${localize.DataScience.untitledNotebookFileName()}-${this.nextNumber}`); - this.nextNumber += 1; - return result; - } - /** * Open ipynb files when user opens an ipynb file. * @@ -205,6 +180,31 @@ export class NativeEditorProvider implements INotebookEditorProvider, IAsyncDisp this.activeEditors.set(e.file.fsPath, e); } + private async getNextNewNotebookUri(): Promise { + // Start in the root and look for files starting with untitled + let number = 1; + const dir = this.workspace.rootPath; + if (dir) { + const existing = await this.fileSystem.search(path.join(dir, `${localize.DataScience.untitledNotebookFileName()}-*.ipynb`)); + + // Sort by number + existing.sort(); + + // Add one onto the end of the last one + if (existing.length > 0) { + const match = /(\w+)-(\d+)\.ipynb/.exec(path.basename(existing[existing.length - 1])); + if (match && match.length > 1) { + number = parseInt(match[2], 10); + } + return Uri.file(path.join(dir, `${localize.DataScience.untitledNotebookFileName()}-${number + 1}`)); + } + } + + const result = Uri.file(`${localize.DataScience.untitledNotebookFileName()}-${this.nextNumber}`); + this.nextNumber += 1; + return result; + } + private openNotebookAndCloseEditor = async (document: TextDocument, closeDocumentBeforeOpeningNotebook: boolean) => { // See if this is an ipynb file if (this.isNotebook(document) && this.configuration.getSettings().datascience.useNotebookEditor && diff --git a/src/client/datascience/jupyter/jupyterExporter.ts b/src/client/datascience/jupyter/jupyterExporter.ts index 0a5a1f6bec2e..89fed8dfbbf5 100644 --- a/src/client/datascience/jupyter/jupyterExporter.ts +++ b/src/client/datascience/jupyter/jupyterExporter.ts @@ -89,7 +89,7 @@ export class JupyterExporter implements INotebookExporter { id: uuid(), file: Identifiers.EmptyFileName, line: 0, - state: CellState.finished, + state: CellState.finished }; return [cell, ...cells]; diff --git a/src/client/datascience/types.ts b/src/client/datascience/types.ts index 7cc572a3f52b..5a34298e544a 100644 --- a/src/client/datascience/types.ts +++ b/src/client/datascience/types.ts @@ -247,7 +247,6 @@ export interface INotebookEditorProvider { show(file: Uri): Promise; createNew(contents?: string): Promise; getNotebookOptions(): Promise; - getNextNewNotebookUri(): Promise; } // For native editing, the INotebookEditor acts like a TextEditor and a TextDocument together diff --git a/src/test/datascience/testNativeEditorProvider.ts b/src/test/datascience/testNativeEditorProvider.ts index d9e2b7b07733..5a92a275b17c 100644 --- a/src/test/datascience/testNativeEditorProvider.ts +++ b/src/test/datascience/testNativeEditorProvider.ts @@ -95,8 +95,4 @@ export class TestNativeEditorProvider implements INotebookEditorProvider { public async getNotebookOptions(): Promise { return this.realProvider.getNotebookOptions(); } - - public async getNextNewNotebookUri(): Promise { - return Uri.file(''); - } } From 17001d63e3eff6923d32b724876bfbcf3f04ac10 Mon Sep 17 00:00:00 2001 From: Jim Griesmer Date: Thu, 31 Oct 2019 17:15:28 -0700 Subject: [PATCH 21/42] Fixups --- src/client/datascience/gather/gather.ts | 4 ++-- src/datascience-ui/native-editor/nativeCell.tsx | 12 +++++------- src/datascience-ui/native-editor/nativeEditor.tsx | 2 +- src/test/datascience/gather/gather.unit.test.ts | 2 +- 4 files changed, 9 insertions(+), 11 deletions(-) diff --git a/src/client/datascience/gather/gather.ts b/src/client/datascience/gather/gather.ts index dc23235bbce9..e1cc0c3bffa5 100644 --- a/src/client/datascience/gather/gather.ts +++ b/src/client/datascience/gather/gather.ts @@ -21,7 +21,7 @@ import { CellState, ICell as IVscCell, IGatherExecution, INotebookExecutionLogge */ @injectable() export class GatherExecution implements IGatherExecution, INotebookExecutionLogger { - private _executionSlicer: ExecutionLogSlicer; + private _executionSlicer: ExecutionLogSlicer; private dataflowAnalyzer: DataflowAnalyzer; private _enabled: boolean; @@ -86,7 +86,7 @@ export class GatherExecution implements IGatherExecution, INotebookExecutionLogg const defaultCellMarker = this.configService.getSettings().datascience.defaultCellMarker || Identifiers.DefaultCodeCellMarker; // Call internal slice method - const slices = this._executionSlicer.sliceAllExecutions(gatherCell.persistentId); + const slices = this._executionSlicer.sliceAllExecutions(gatherCell); const program = slices.length > 0 ? slices[0].cellSlices.reduce(concat, '').replace(/#%%/g, defaultCellMarker) : ''; // Add a comment at the top of the file explaining what gather does diff --git a/src/datascience-ui/native-editor/nativeCell.tsx b/src/datascience-ui/native-editor/nativeCell.tsx index bd50f5505c3c..536eb9ff94b4 100644 --- a/src/datascience-ui/native-editor/nativeCell.tsx +++ b/src/datascience-ui/native-editor/nativeCell.tsx @@ -574,12 +574,10 @@ export class NativeCell extends React.Component { this.props.stateController.runBelow(cellId); this.props.stateController.sendCommand(NativeCommandType.RunBelow, 'mouse'); }; - const gatherDisabled = () => { - return this.props.cellVM.cell.data.execution_count === null || - this.props.cellVM.hasBeenRun === null || - this.props.cellVM.hasBeenRun === false || - getSettings().enableGather === false; - }; + const gatherDisabled = this.props.cellVM.cell.data.execution_count === null || + this.props.cellVM.hasBeenRun === null || + this.props.cellVM.hasBeenRun === false || + getSettings().enableGather === false; const canRunAbove = this.props.stateController.canRunAbove(cellId); const canRunBelow = this.props.cellVM.cell.state === CellState.finished || this.props.cellVM.cell.state === CellState.error; const switchTooltip = this.props.cellVM.cell.data.cell_type === 'code' ? getLocString('DataScience.switchToMarkdown', 'Change to markdown') : @@ -606,7 +604,7 @@ export class NativeCell extends React.Component { -