"use strict"; const {Fragment: _Fragment, jsx: _jsx, jsxs: _jsxs} = arguments[0]; const {useMDXComponents: _provideComponents} = arguments[0]; function _createMdxContent(props) { const _components = { a: "a", blockquote: "blockquote", code: "code", em: "em", h1: "h1", h2: "h2", h3: "h3", li: "li", mermaid: "mermaid", ol: "ol", p: "p", path: "path", strong: "strong", svg: "svg", ul: "ul", ..._provideComponents(), ...props.components }; return _jsxs(_Fragment, { children: [_jsxs(_components.h1, { id: "motivation-roadmap-scqa-ii---augsep-2017", children: [_jsx(_components.a, { className: "heading-link", href: "#motivation-roadmap-scqa-ii---augsep-2017", children: _jsx(_components.svg, { xmlns: "http:www.w3.org/2000/svg", fill: "#ab2b65", viewBox: "0 0 20 20", className: "w-5 h-5", children: _jsx(_components.path, { fillRule: "evenodd", clipRule: "evenodd", d: "M9.493 2.853a.75.75 0 00-1.486-.205L7.545 6H4.198a.75.75 0 000 1.5h3.14l-.69 5H3.302a.75.75 0 000 1.5h3.14l-.435 3.148a.75.75 0 001.486.205L7.955 14h2.986l-.434 3.148a.75.75 0 001.486.205L12.456 14h3.346a.75.75 0 000-1.5h-3.14l.69-5h3.346a.75.75 0 000-1.5h-3.14l.435-3.147a.75.75 0 00-1.486-.205L12.045 6H9.059l.434-3.147zM8.852 7.5l-.69 5h2.986l.69-5H8.852z" }) }) }), "Motivation: Roadmap SCQA II - Aug/Sep 2017"] }), "\n", _jsx(_components.p, { children: "Situation: alpha running, no publishers yet, 50 signed up users." }), "\n", _jsx(_components.p, { children: "Complication: we don't have any paying users yet (or know whether we will) and we don't have a clear plan of action." }), "\n", _jsx(_components.p, { children: "Question: what tasks should we focus on in the next 3-6m to create a sustainable business within the next 6-12m?" }), "\n", _jsxs(_components.ul, { children: ["\n", _jsx(_components.li, { children: "what tasks features [user needs] and tasks should we focus on to create a sustainable business" }), "\n", _jsx(_components.li, { children: "How do we create a sustainable business?" }), "\n"] }), "\n", _jsx(_components.p, { children: "Hypothesis: Focus on three complementary areas: provide a growing set of core data for consumers using our own tooling to automate this; provide an initial offering to datars focused on frictionless inspecting and sharing; and engage in \"lean\" market testing to test and validate products offers." }), "\n", _jsxs(_components.ul, { children: ["\n", _jsxs(_components.li, { children: [_jsx(_components.strong, { children: "Consumers - core data" }), " Solidify and expand our core dataset offering in order to attract datars and validate consumer hypothesis (and eat our own dog food)", "\n", _jsxs(_components.ul, { children: ["\n", _jsxs(_components.li, { children: ["Develop workflows ", _jsx(_components.em, { children: "inside DataHub" }), " that make this cheap and automated (esp for datasets that change)"] }), "\n"] }), "\n"] }), "\n", _jsxs(_components.li, { children: [_jsx(_components.strong, { children: "Marketing / research" }), " Put out \"tasters\" for main business offerings to see who is interested (e.g. enterprise, request data, FD consulting …)", "\n", _jsxs(_components.ul, { children: ["\n", _jsx(_components.li, { children: "Do surveys, write up vision and use to inform how we write to people" }), "\n"] }), "\n"] }), "\n", _jsxs(_components.li, { children: [_jsx(_components.strong, { children: "Push - datars" }), " Focus on datars and make it simple and easy to push and pull data automatically and charge for private datasets plus storage over a certain level"] }), "\n"] }), "\n", _jsx(_components.p, { children: _jsx(_components.em, { children: "This got reworked - see next item" }) }), "\n", _jsxs(_components.h1, { id: "roadmap-sep-2017", children: [_jsx(_components.a, { className: "heading-link", href: "#roadmap-sep-2017", children: _jsx(_components.svg, { xmlns: "http:www.w3.org/2000/svg", fill: "#ab2b65", viewBox: "0 0 20 20", className: "w-5 h-5", children: _jsx(_components.path, { fillRule: "evenodd", clipRule: "evenodd", d: "M9.493 2.853a.75.75 0 00-1.486-.205L7.545 6H4.198a.75.75 0 000 1.5h3.14l-.69 5H3.302a.75.75 0 000 1.5h3.14l-.435 3.148a.75.75 0 001.486.205L7.955 14h2.986l-.434 3.148a.75.75 0 001.486.205L12.456 14h3.346a.75.75 0 000-1.5h-3.14l.69-5h3.346a.75.75 0 000-1.5h-3.14l.435-3.147a.75.75 0 00-1.486-.205L12.045 6H9.059l.434-3.147zM8.852 7.5l-.69 5h2.986l.69-5H8.852z" }) }) }), "Roadmap (Sep 2017)"] }), "\n", _jsx(_components.p, { children: "Hypothesis: Focus on three complementary areas:" }), "\n", _jsxs(_components.ul, { children: ["\n", _jsxs(_components.li, { children: [_jsx(_components.strong, { children: "Core datasets" }), ": Solidify and expand our core datasets using our own tooling to automate this in order to eat our own dog food, attract datars and validate consumer hypothesis", "\n", _jsxs(_components.ul, { children: ["\n", _jsxs(_components.li, { children: ["Develop workflows ", _jsx(_components.em, { children: "inside DataHub" }), " that make this cheap and automated (esp for datasets that change)"] }), "\n"] }), "\n"] }), "\n", _jsxs(_components.li, { children: [_jsx(_components.strong, { children: "Marketing / research" }), ": engage in lean marketing to test and validate product offers by putting out \"tasters\" for main business offerings to see who is interested (e.g. enterprise, request data, FD consulting …)", "\n", _jsxs(_components.ul, { children: ["\n", _jsx(_components.li, { children: "Do surveys, write up vision and use to inform how we write to people" }), "\n"] }), "\n"] }), "\n", _jsx(_components.li, { children: _jsxs(_components.em, { children: ["[SECONDARY] ", _jsx(_components.strong, { children: "Push - datars" }), " make it simple and easy for datars to push and pull data automatically and charge for private datasets plus storage over a certain level"] }) }), "\n"] }), "\n", _jsxs(_components.h1, { id: "detailed-plan", children: [_jsx(_components.a, { className: "heading-link", href: "#detailed-plan", children: _jsx(_components.svg, { xmlns: "http:www.w3.org/2000/svg", fill: "#ab2b65", viewBox: "0 0 20 20", className: "w-5 h-5", children: _jsx(_components.path, { fillRule: "evenodd", clipRule: "evenodd", d: "M9.493 2.853a.75.75 0 00-1.486-.205L7.545 6H4.198a.75.75 0 000 1.5h3.14l-.69 5H3.302a.75.75 0 000 1.5h3.14l-.435 3.148a.75.75 0 001.486.205L7.955 14h2.986l-.434 3.148a.75.75 0 001.486.205L12.456 14h3.346a.75.75 0 000-1.5h-3.14l.69-5h3.346a.75.75 0 000-1.5h-3.14l.435-3.147a.75.75 0 00-1.486-.205L12.045 6H9.059l.434-3.147zM8.852 7.5l-.69 5h2.986l.69-5H8.852z" }) }) }), "Detailed Plan"] }), "\n", _jsxs(_components.h2, { id: "marketing-roadmap", children: [_jsx(_components.a, { className: "heading-link", href: "#marketing-roadmap", children: _jsx(_components.svg, { xmlns: "http:www.w3.org/2000/svg", fill: "#ab2b65", viewBox: "0 0 20 20", className: "w-5 h-5", children: _jsx(_components.path, { fillRule: "evenodd", clipRule: "evenodd", d: "M9.493 2.853a.75.75 0 00-1.486-.205L7.545 6H4.198a.75.75 0 000 1.5h3.14l-.69 5H3.302a.75.75 0 000 1.5h3.14l-.435 3.148a.75.75 0 001.486.205L7.955 14h2.986l-.434 3.148a.75.75 0 001.486.205L12.456 14h3.346a.75.75 0 000-1.5h-3.14l.69-5h3.346a.75.75 0 000-1.5h-3.14l.435-3.147a.75.75 0 00-1.486-.205L12.045 6H9.059l.434-3.147zM8.852 7.5l-.69 5h2.986l.69-5H8.852z" }) }) }), "Marketing Roadmap"] }), "\n", _jsx(_components.p, { children: _jsx(_components.em, { children: "Less defined. Added retrospectively …" }) }), "\n", _jsxs(_components.h2, { id: "developer-roadmap", children: [_jsx(_components.a, { className: "heading-link", href: "#developer-roadmap", children: _jsx(_components.svg, { xmlns: "http:www.w3.org/2000/svg", fill: "#ab2b65", viewBox: "0 0 20 20", className: "w-5 h-5", children: _jsx(_components.path, { fillRule: "evenodd", clipRule: "evenodd", d: "M9.493 2.853a.75.75 0 00-1.486-.205L7.545 6H4.198a.75.75 0 000 1.5h3.14l-.69 5H3.302a.75.75 0 000 1.5h3.14l-.435 3.148a.75.75 0 001.486.205L7.955 14h2.986l-.434 3.148a.75.75 0 001.486.205L12.456 14h3.346a.75.75 0 000-1.5h-3.14l.69-5h3.346a.75.75 0 000-1.5h-3.14l.435-3.147a.75.75 0 00-1.486-.205L12.045 6H9.059l.434-3.147zM8.852 7.5l-.69 5h2.986l.69-5H8.852z" }) }) }), "Developer Roadmap"] }), "\n", _jsx(_components.p, { children: "NTS 2017-09-17" }), "\n", _jsxs(_components.ul, { children: ["\n", _jsxs(_components.li, { children: ["Export formats and showcase – lots of options", "\n", _jsxs(_components.ul, { children: ["\n", _jsx(_components.li, { children: "This is an obvious point for external developers to contribute" }), "\n", _jsx(_components.li, { children: "=> Defining this contribution \"endpoint\" - creating drivers" }), "\n"] }), "\n"] }), "\n", _jsxs(_components.li, { children: ["Import + Transform", "\n", _jsxs(_components.ul, { children: ["\n", _jsxs(_components.li, { children: ["Driven: part the core datasets", "\n", _jsxs(_components.ul, { children: ["\n", _jsx(_components.li, { children: "Add features needed to support them one by one" }), "\n"] }), "\n"] }), "\n", _jsx(_components.li, { children: "Do by hand and push" }), "\n", _jsx(_components.li, { children: "Then infer …" }), "\n"] }), "\n"] }), "\n", _jsx(_components.li, { children: "Interfaces e.g. CLI, Local App, Web interface" }), "\n", _jsxs(_components.li, { children: ["Management features", "\n", _jsxs(_components.ul, { children: ["\n", _jsx(_components.li, { children: "Usage tracking" }), "\n", _jsx(_components.li, { children: "Privacy" }), "\n", _jsx(_components.li, { children: "Teams" }), "\n"] }), "\n"] }), "\n", _jsxs(_components.li, { children: ["Getting paid", "\n", _jsxs(_components.ul, { children: ["\n", _jsx(_components.li, { children: "Integrating payment" }), "\n", _jsx(_components.li, { children: "Supporting users" }), "\n", _jsx(_components.li, { children: "Marketing" }), "\n"] }), "\n"] }), "\n"] }), "\n", _jsx(_components.mermaid, { config: "{}", chart: "graph LR\n\nsubgraph Automated Clean\ncoreauto[Core Auto v0.1 - 2 pkgs

Write specs by hand & push]\ncoreautov0.2[Core Auto v0.2 - 10 pkgs]\nend\n\nsubgraph Export\nexport1[Export Sqlite, Zip]\nexport2[Export Node/npm]\nexport3[Make your own exporter

involve community in
creating them]\nend\n\nsubgraph Push UI\nclipush[CLI - Push
url/Push spec]\ncliv0.6[\"CLI Tidy (minimal)\"]\nend\n\nfactoryv2(Factory v2 Redesign
DAG + Parallelization)\nshowcase[Showcase v1.1
download UX, R instructions]\npreview[Preview large files]\n\nsubgraph Tracking\ntrack[Tracking

Storage, B/W, CPU]\ntrackv0.1[Track usage - storage]\ntrackv0.2[Track usage - downloads]\ntrackv0.3[Track usage - cpu]\nend\n\nstorechanges(Store Every Change)\neventstore(EventStore)\ndashboard[Dashboard v1]\nvalidate[Validation
Pipeline Reports]\n\nfactoryv2 --> coreauto\ncoreauto --> coreautov0.2\n\nfactoryv2 --> export1\nfactoryv2 --> preview\n\nexport1 --> export2\nexport2 --> export3\n\nclipush --> cliv0.6\nclipush --> coreauto\n\npreview --> showcase\n\ntrackv0.2 --> showcase\ntrackv0.2 --> track\nstorechanges --> trackv0.1\ntrackv0.1 --> showcase\ntrackv0.1 --> track\ntrackv0.3 --> track\n\neventstore --> dashboard\n\nclassDef donemidnov fill:lightgreen,stroke:#333,stroke-width:2px;\nclassDef donedec fill:#7BBF6A,stroke:#333,stroke-width:2px;\nclassDef priority fill:pink,stroke:#333,stroke-width:2px;\nclass factoryv2,coreauto,preview,trackv0.1,clipush,export1 donemidnov;\nclass eventstore,coreautov0.2,cliv0.6,showcase,dashboard,validate,storechanges donedec;" }), "\n", _jsx(_components.p, { children: "Developer …" }), "\n", _jsxs(_components.ul, { children: ["\n", _jsx(_components.li, { children: "EventStore: an index in MetaStore where we store user related events e.g. runs, user sign ins, dataset created, downloads (?)" }), "\n", _jsx(_components.li, { children: "Store every change: new model for storing outputs of every flow run in an economical way - see TODO" }), "\n", _jsx(_components.li, { children: "Economising on version space …" }), "\n", _jsx(_components.li, { children: "Research electron multi-platform builds" }), "\n"] }), "\n", _jsx(_components.p, { children: "Language" }), "\n", _jsxs(_components.ul, { children: ["\n", _jsx(_components.li, { children: "Assembler" }), "\n", _jsx(_components.li, { children: "Assembly line vs flow …" }), "\n", _jsx(_components.li, { children: "Assembly spec (blueprint?)" }), "\n"] }), "\n", _jsxs(_components.h2, { id: "features", children: [_jsx(_components.a, { className: "heading-link", href: "#features", children: _jsx(_components.svg, { xmlns: "http:www.w3.org/2000/svg", fill: "#ab2b65", viewBox: "0 0 20 20", className: "w-5 h-5", children: _jsx(_components.path, { fillRule: "evenodd", clipRule: "evenodd", d: "M9.493 2.853a.75.75 0 00-1.486-.205L7.545 6H4.198a.75.75 0 000 1.5h3.14l-.69 5H3.302a.75.75 0 000 1.5h3.14l-.435 3.148a.75.75 0 001.486.205L7.955 14h2.986l-.434 3.148a.75.75 0 001.486.205L12.456 14h3.346a.75.75 0 000-1.5h-3.14l.69-5h3.346a.75.75 0 000-1.5h-3.14l.435-3.147a.75.75 0 00-1.486-.205L12.045 6H9.059l.434-3.147zM8.852 7.5l-.69 5h2.986l.69-5H8.852z" }) }) }), "Features"] }), "\n", _jsx(_components.p, { children: "Priorities" }), "\n", _jsxs(_components.ul, { children: ["\n", _jsxs(_components.li, { children: ["Experiment and evaluate with \"clean/transform\" automation", "\n", _jsxs(_components.ul, { children: ["\n", _jsx(_components.li, { children: "Automate: cleaning 5-10 core datasets" }), "\n", _jsx(_components.li, { children: "Review the core datasets - do they require regular updating, how does update currently work" }), "\n", _jsx(_components.li, { children: "=> Product definition on left hand side: UI for a automated wrangling system (that would allow us to compare this to other potential product visions …)" }), "\n"] }), "\n"] }), "\n", _jsxs(_components.li, { children: ["Export: npm, zip, sqlite, rds", "\n", _jsxs(_components.ul, { children: ["\n", _jsx(_components.li, { children: "Fix R, python integration instructions …" }), "\n"] }), "\n"] }), "\n", _jsx(_components.li, { children: "Performance and tracking: work to speed things up etc, track usage, gather metrics about what is working …" }), "\n", _jsx(_components.li, { children: "Marketing: …" }), "\n", _jsxs(_components.li, { children: ["CLI", "\n", _jsxs(_components.ul, { children: ["\n", _jsx(_components.li, { children: "progress bar …" }), "\n", _jsx(_components.li, { children: "don't reupload if data already there …" }), "\n", _jsx(_components.li, { children: "lots more" }), "\n"] }), "\n"] }), "\n", _jsx(_components.li, { children: "Docs" }), "\n", _jsxs(_components.li, { children: ["Comms & marketing", "\n", _jsxs(_components.ul, { children: ["\n", _jsx(_components.li, { children: "Pricing - and offer" }), "\n", _jsx(_components.li, { children: "Write posts about datasets" }), "\n", _jsx(_components.li, { children: "About tools .." }), "\n"] }), "\n"] }), "\n"] }), "\n", _jsx(_components.p, { children: "NTS:" }), "\n", _jsx(_components.p, { children: "Question: how do we connection \"pipelines\" together (concept of the dataset or project)" }), "\n", _jsxs(_components.h3, { id: "more", children: [_jsx(_components.a, { className: "heading-link", href: "#more", children: _jsx(_components.svg, { xmlns: "http:www.w3.org/2000/svg", fill: "#ab2b65", viewBox: "0 0 20 20", className: "w-5 h-5", children: _jsx(_components.path, { fillRule: "evenodd", clipRule: "evenodd", d: "M9.493 2.853a.75.75 0 00-1.486-.205L7.545 6H4.198a.75.75 0 000 1.5h3.14l-.69 5H3.302a.75.75 0 000 1.5h3.14l-.435 3.148a.75.75 0 001.486.205L7.955 14h2.986l-.434 3.148a.75.75 0 001.486.205L12.456 14h3.346a.75.75 0 000-1.5h-3.14l.69-5h3.346a.75.75 0 000-1.5h-3.14l.435-3.147a.75.75 0 00-1.486-.205L12.045 6H9.059l.434-3.147zM8.852 7.5l-.69 5h2.986l.69-5H8.852z" }) }) }), "More"] }), "\n", _jsxs(_components.ul, { children: ["\n", _jsxs(_components.li, { children: ["set privacy", "\n", _jsxs(_components.ul, { children: ["\n", _jsx(_components.li, { children: "change source …" }), "\n", _jsx(_components.li, { children: "explicitly set public and private or …?" }), "\n"] }), "\n"] }), "\n", _jsx(_components.li, { children: "support charging" }), "\n", _jsx(_components.li, { children: "preview large datasets" }), "\n", _jsx(_components.li, { children: "introducing data validation (goodtables lite)" }), "\n", _jsxs(_components.li, { children: ["data statistics: percentage complete ", _jsx(_components.a, { href: "https://github.com/ResidentMario/missingno", target: "_blank", rel: "noopener noreferrer", children: "https://github.com/ResidentMario/missingno" })] }), "\n", _jsxs(_components.li, { children: ["Import from URL: ", _jsx(_components.code, { className: "language-auto", children: "data add {url} --cron=1d" })] }), "\n", _jsx(_components.li, { children: "cron job scraping" }), "\n", _jsxs(_components.li, { children: ["modify / add views", "\n", _jsxs(_components.ul, { children: ["\n", _jsx(_components.li, { children: "And the preview: preprocess the transform" }), "\n"] }), "\n"] }), "\n", _jsxs(_components.li, { children: ["Transforms on data", "\n", _jsxs(_components.ul, { children: ["\n", _jsx(_components.li, { children: "=> transform language …" }), "\n"] }), "\n"] }), "\n", _jsx(_components.li, { children: "edit title" }), "\n", _jsxs(_components.li, { children: ["debug pipeline", "\n", _jsxs(_components.ul, { children: ["\n", _jsx(_components.li, { children: "view the pipeline structure and data at each stage" }), "\n"] }), "\n"] }), "\n", _jsx(_components.li, { children: "push to bigquery" }), "\n", _jsxs(_components.li, { children: ["showcase improvements", "\n", _jsxs(_components.ul, { children: ["\n", _jsxs(_components.li, { children: ["/v/", viewid, ".png or /v/", viewid, ".html?embed=true", "\n", _jsxs(_components.ul, { children: ["\n", _jsx(_components.li, { children: "Why: so you can embed" }), "\n"] }), "\n"] }), "\n", _jsx(_components.li, { children: "view json (?)" }), "\n", _jsxs(_components.li, { children: ["/r/", resource, "/", primary - key - id, ".[json|html|csv]", "\n", _jsxs(_components.ul, { children: ["\n", _jsx(_components.li, { children: "e.g. /data/population/r/annual/2010-us" }), "\n"] }), "\n"] }), "\n"] }), "\n"] }), "\n"] }), "\n", _jsxs(_components.h1, { id: "evaluation-of-q4-2017-mid-dec-2017", children: [_jsx(_components.a, { className: "heading-link", href: "#evaluation-of-q4-2017-mid-dec-2017", children: _jsx(_components.svg, { xmlns: "http:www.w3.org/2000/svg", fill: "#ab2b65", viewBox: "0 0 20 20", className: "w-5 h-5", children: _jsx(_components.path, { fillRule: "evenodd", clipRule: "evenodd", d: "M9.493 2.853a.75.75 0 00-1.486-.205L7.545 6H4.198a.75.75 0 000 1.5h3.14l-.69 5H3.302a.75.75 0 000 1.5h3.14l-.435 3.148a.75.75 0 001.486.205L7.955 14h2.986l-.434 3.148a.75.75 0 001.486.205L12.456 14h3.346a.75.75 0 000-1.5h-3.14l.69-5h3.346a.75.75 0 000-1.5h-3.14l.435-3.147a.75.75 0 00-1.486-.205L12.045 6H9.059l.434-3.147zM8.852 7.5l-.69 5h2.986l.69-5H8.852z" }) }) }), "Evaluation of Q4 2017 (mid Dec 2017)"] }), "\n", _jsx(_components.p, { children: "Hypothesis: Focus on three complementary areas:" }), "\n", _jsxs(_components.ul, { children: ["\n", _jsxs(_components.li, { children: [_jsx(_components.strong, { children: "Core datasets" }), ": Solidify and expand our core datasets using our own tooling to automate this in order to eat our own dog food, attract datars and validate consumer hypothesis", "\n", _jsxs(_components.ul, { children: ["\n", _jsxs(_components.li, { children: ["Develop workflows ", _jsx(_components.em, { children: "inside DataHub" }), " that make this cheap and automated (esp for datasets that change)"] }), "\n"] }), "\n"] }), "\n"] }), "\n", _jsxs(_components.blockquote, { children: ["\n", _jsx(_components.p, { children: "Outcome: 70% - core dataset automation so far is only about 10% done. Current priority and by end of year we may have advanced substantially towards what we want." }), "\n", _jsx(_components.p, { children: "Available to users but undocumented" }), "\n"] }), "\n", _jsxs(_components.ul, { children: ["\n", _jsxs(_components.li, { children: [_jsx(_components.strong, { children: "Marketing / research" }), ": engage in lean marketing to test and validate product offers by putting out \"tasters\" for main business offerings to see who is interested (e.g. enterprise, request data, FD consulting …)", "\n", _jsxs(_components.ul, { children: ["\n", _jsx(_components.li, { children: "Do surveys, write up vision and use to inform how we write to people" }), "\n"] }), "\n"] }), "\n"] }), "\n", _jsxs(_components.blockquote, { children: ["\n", _jsx(_components.p, { children: "Outcome: 30%." }), "\n", _jsxs(_components.ul, { children: ["\n", _jsx(_components.li, { children: "Request data done" }), "\n", _jsx(_components.li, { children: "Pricing done" }), "\n", _jsx(_components.li, { children: "Did not do FD consulting and enterprise." }), "\n", _jsx(_components.li, { children: "Blog posts and rising traffic" }), "\n"] }), "\n"] }), "\n", _jsxs(_components.ul, { children: ["\n", _jsx(_components.li, { children: _jsxs(_components.em, { children: ["[SECONDARY] ", _jsx(_components.strong, { children: "Push - datars" }), " make it simple and easy for datars to push and pull data automatically and charge for private datasets plus storage over a certain level"] }) }), "\n"] }), "\n", _jsxs(_components.blockquote, { children: ["\n", _jsx(_components.p, { children: "Outcome: 80%." }), "\n", _jsxs(_components.ul, { children: ["\n", _jsx(_components.li, { children: "Implemented pricing" }), "\n", _jsx(_components.li, { children: "Implemented automated pull of data (but not super well promoted)" }), "\n"] }), "\n"] }), "\n", _jsx(_components.p, { children: "Extras" }), "\n", _jsxs(_components.ul, { children: ["\n", _jsx(_components.li, { children: "We implemented data validation so that we can validate it as a product" }), "\n", _jsx(_components.li, { children: "We completely rebuilt our data factory which massively improved performance" }), "\n"] }), "\n", _jsxs(_components.h1, { id: "appendix", children: [_jsx(_components.a, { className: "heading-link", href: "#appendix", children: _jsx(_components.svg, { xmlns: "http:www.w3.org/2000/svg", fill: "#ab2b65", viewBox: "0 0 20 20", className: "w-5 h-5", children: _jsx(_components.path, { fillRule: "evenodd", clipRule: "evenodd", d: "M9.493 2.853a.75.75 0 00-1.486-.205L7.545 6H4.198a.75.75 0 000 1.5h3.14l-.69 5H3.302a.75.75 0 000 1.5h3.14l-.435 3.148a.75.75 0 001.486.205L7.955 14h2.986l-.434 3.148a.75.75 0 001.486.205L12.456 14h3.346a.75.75 0 000-1.5h-3.14l.69-5h3.346a.75.75 0 000-1.5h-3.14l.435-3.147a.75.75 0 00-1.486-.205L12.045 6H9.059l.434-3.147zM8.852 7.5l-.69 5h2.986l.69-5H8.852z" }) }) }), "Appendix"] }), "\n", _jsxs(_components.h2, { id: "discussion-3-sep-2017---rufus--adam", children: [_jsx(_components.a, { className: "heading-link", href: "#discussion-3-sep-2017---rufus--adam", children: _jsx(_components.svg, { xmlns: "http:www.w3.org/2000/svg", fill: "#ab2b65", viewBox: "0 0 20 20", className: "w-5 h-5", children: _jsx(_components.path, { fillRule: "evenodd", clipRule: "evenodd", d: "M9.493 2.853a.75.75 0 00-1.486-.205L7.545 6H4.198a.75.75 0 000 1.5h3.14l-.69 5H3.302a.75.75 0 000 1.5h3.14l-.435 3.148a.75.75 0 001.486.205L7.955 14h2.986l-.434 3.148a.75.75 0 001.486.205L12.456 14h3.346a.75.75 0 000-1.5h-3.14l.69-5h3.346a.75.75 0 000-1.5h-3.14l.435-3.147a.75.75 0 00-1.486-.205L12.045 6H9.059l.434-3.147zM8.852 7.5l-.69 5h2.986l.69-5H8.852z" }) }) }), "Discussion 3 Sep 2017 - Rufus + Adam"] }), "\n", _jsx(_components.p, { children: "\"Pipelines\" can be used for:" }), "\n", _jsxs(_components.ul, { children: ["\n", _jsx(_components.li, { children: "cleaning: \"…\"" }), "\n", _jsx(_components.li, { children: "transform: \"normal pipelines\"" }), "\n", _jsx(_components.li, { children: "export aka \"dump\"" }), "\n"] }), "\n", _jsx(_components.p, { children: "Discussion:" }), "\n", _jsxs(_components.ul, { children: ["\n", _jsx(_components.li, { children: "Where do we focus in the data flow (for now)" }), "\n", _jsxs(_components.li, { children: ["Do we focus on clean, on transform or export?", "\n", _jsxs(_components.ul, { children: ["\n", _jsx(_components.li, { children: "Adam: there isn't a lot of benefit of pushing a static data file directly to bigquery - it's not that interesting …, it's a bit trivial" }), "\n", _jsx(_components.li, { children: "Rufus: good point, and partly that's why we emphasise showcase so much?" }), "\n", _jsx(_components.li, { children: "what's gives value to users and what can we get money from?" }), "\n", _jsxs(_components.li, { children: ["Rufus: it is a tough sell for people to get automated cleaning - because they have to understand ", _jsx(_components.em, { children: "and" }), " adopt your system"] }), "\n"] }), "\n"] }), "\n", _jsxs(_components.li, { children: ["Proposal", "\n", _jsxs(_components.ul, { children: ["\n", _jsxs(_components.li, { children: ["Experiment and evaluate with \"clean/transform\" automation", "\n", _jsxs(_components.ul, { children: ["\n", _jsx(_components.li, { children: "Automate: cleaning 5-10 core datasets" }), "\n", _jsx(_components.li, { children: "Review the core datasets - do they require regular updating, how does update currently work" }), "\n", _jsx(_components.li, { children: "=> Product definition on left hand side: UI for a automated wrangling system (that would allow us to compare this to other potential product visions …)" }), "\n"] }), "\n"] }), "\n", _jsxs(_components.li, { children: ["Export: npm, zip, sqlite, rds", "\n", _jsxs(_components.ul, { children: ["\n", _jsx(_components.li, { children: "Fix R, python integration instructions …" }), "\n"] }), "\n"] }), "\n", _jsx(_components.li, { children: "Performance and tracking: work to speed things up etc, track usage, gather metrics about what is working …" }), "\n", _jsx(_components.li, { children: "Marketing: …" }), "\n"] }), "\n"] }), "\n"] }), "\n", _jsx(_components.p, { children: "Product ideas" }), "\n", _jsxs(_components.ul, { children: ["\n", _jsx(_components.li, { children: "Export to your own RDS …" }), "\n", _jsx(_components.li, { children: "Metaphor of \"deployment\"" }), "\n"] }), "\n", _jsx(_components.p, { children: "If we want to use pipelines for cleaning:" }), "\n", _jsxs(_components.ul, { children: ["\n", _jsxs(_components.li, { children: ["DSL to define clean operation (delete first two rows, do this regex etc)", "\n", _jsxs(_components.ul, { children: ["\n", _jsx(_components.li, { children: "?? - has significant limits - not clear what 80/20 looks like (?) - \"this way lies madness\" ;-)" }), "\n"] }), "\n"] }), "\n", _jsxs(_components.li, { children: ["\"Dockerize\" or DPP-ize the operation: allow people to write code either any kind of code (docker) or new pipelines that: raw data -> data package", "\n", _jsxs(_components.ul, { children: ["\n", _jsx(_components.li, { children: "=> registry for \"processors\" (be they docker or a pipelines)" }), "\n"] }), "\n"] }), "\n"] }), "\n", _jsx(_components.p, { children: "Golden path" }), "\n", _jsxs(_components.ol, { children: ["\n", _jsxs(_components.li, { children: ["Simple cleaning operations DSL e.g. Borrow from unix shell", "\n", _jsxs(_components.ul, { children: ["\n", _jsx(_components.li, { children: _jsx(_components.a, { href: "https://github.com/rufuspollock/command-line-data-wrangling", target: "_blank", rel: "noopener noreferrer", children: "https://github.com/rufuspollock/command-line-data-wrangling" }) }), "\n", _jsx(_components.li, { children: _jsx(_components.a, { href: "http://datapipes.okfnlabs.org/", target: "_blank", rel: "noopener noreferrer", children: "http://datapipes.okfnlabs.org/" }) }), "\n"] }), "\n"] }), "\n", _jsxs(_components.li, { children: ["Allow for snippets (leading to full blown pipelines)", "\n", _jsxs(_components.ul, { children: ["\n", _jsx(_components.li, { children: "(middle way) defined limited API" }), "\n"] }), "\n"] }), "\n"] })] }); } function MDXContent(props = {}) { const {wrapper: MDXLayout} = { ..._provideComponents(), ...props.components }; return MDXLayout ? _jsx(MDXLayout, { ...props, children: _jsx(_createMdxContent, { ...props }) }) : _createMdxContent(props); } return { default: MDXContent };