feat(infra) Rebuild hyperlink (#2382)

* Update link checking to use latest hyperlink. Replaces #1582

* Stop appending empty fragment to urls pointing ad no fragment identifier

* Resolve relative hrefs from README's to their github.com url instead of linking locally inside docs

* Prefer https where available and also use https in examples

* Make non-fragment links inside headings visible

* Update webpack docs china link

* Fixed invalid edit links to index pages

* skip fragment check for vimdoc that is known to fail

* Fix broken link to migration guide from configuration/resolve

* Also resolve github README relative links when they don't start with a dot

* Add missing pages: Vote, Organization, Starter kits

* Remove unused code

* Fix link that was redirected

* Open all target='_blank' links with rel='noopener' for security

* Skip link checking img.shields.io

* Change order of markdown url rewriting to avoid errors

* Use lowercase variable name for node url module. See #2382 (comment)

* Update link to file-loader github repo

* Rewrite links from https://npmjs.com to https://www.npmjs.com to avoid the inevitable redirect

* Skip checking links to npmjs.com/package that causes hyperlink to get http 500 responses. This should probably be reverted and fixed later

* Updated organization projects urls to their redirect target

* Update tool-list to get correct links for projects

* Updated links to their redirect target

* Remove jscs-loader. The project is deprecated, merged with eslint and the domain we link to is for sale

* Update link to karma-webpack project

* Remove link to named-modules-plugin, which is now in webpack core

* Added missing mini-css-extract-plugin to fetch script

* Skip checking links to known deleted github users

* Update broken fragments to work with new markdown generator

* Fixed more broken links

* infra(site) Copy assets when building

* Update github location for css-loader

* Remove dead github accounts from contributors listings

* infra(site) Fix fetchPackages script when running locally

* Fix internal fragmtn links in optimization.md

* Skip link checking or opencollective.com/webpack. Massive html response made the checker go into CPU overdrive

* Remove link to non-existing named-modules-plugin

* Use new release of hyperlink

* feat(infra) Travis optimization (#2404)

* Fix internal fragmtn links in optimization.md

* Skip link checking or opencollective.com/webpack. Massive html response made the checker go into CPU overdrive

* Try out travis staged build

* Add proselint

* Upgrade pip before using it

* Move before-hooks around to try and make proselint install

* Try adding python as a language as well to geet an updated version

* More messing with config

* Manually handle node versioning

* Add node minor version to nvm install. Defaulted to slightly too low version

* Manually install node 8.11

* Try a matrix build to separate node and python stuff

* Add linkcheck task and try a different cahce setup

* Minor name change to test if cache works correctly across builds

* Attempt to combine matrix and staged build

* Attempt going back to staged build

* Bump travis

* Ping Travis. You alive?

* Fix broken travis.yml

* Fix wrong stage order

* Explicitly run specific lintings, exclude proselint

* Allow failures for link checker

* Change proselint cache directory. Maybe this works

* Add new script to fetch repository names that the docs should contain. Try to centralize github API usage to avoid rate limitations

* Add caching to eslint

* Remove parts of deploy.sh that are now run by travis

* Added new ./repositories to store github api results

* Replace fetch.sh with npm scripts and fetch-package-readmes.js

* Attempt to make caches more specific to the containers and stages they refer to

* Remove deprecaed fetch_packages script. Now replaced by two-step fetch-package-repos and fetch-package-readmes

* Attempt a more windows friendly build by using npm and webpack instead of shell commands

* Disable link checking for now to speed up builds

* Revert "Disable link checking for now to speed up builds"

This reverts commit 7d4bb06.

* Add dist to proselint cache so generated content also gets checked

* Remove unnessessary GITHUB_TOKEN env variable check in repo fetching script

* Revert "Add dist to proselint cache so generated content also gets checked"

This reverts commit fc7676d.

* Rework pipeline for better speed and make proselint a deployment blocker

* Rename build stage to reflect its actions

* Run content-tree only after generating all external content

* Remove link to non-existing named-modules-plugin

* Fix double slashes in edit links

* Rename stages

* Add new internal link check as a blocking check

* Fix wrong name in allowed_failures config

.gitignore

@@ -8,3 +8,4 @@ src/**/_*.md
 npm-debug.log
 yarn-error.log
 package-lock.json
+.cache

.travis.yml

@@ -1,15 +1,52 @@
-branches:
-  only:
-    - master
-    - develop
+sudo: false
 language: node_js
 node_js:
   - "8"
-sudo: required
-install:
-  - yarn
-  - sudo pip install proselint
-before_script:
-  - source ./src/scripts/env.sh
-script:
-  - bash ./src/scripts/deploy.sh
+
+cache:
+  yarn: true
+  directories:
+    - node_modules
+    - dist
+    - .cache
+
+stages:
+  - Build
+  - Post-build
+
+jobs:
+  fast_finish: true
+  allow_failures:
+    - name: External Link Check
+
+  include:
+    - stage: Build
+      name: Lint and Build
+      before_install: npm install --global yarn
+      install: yarn
+      script:
+        - yarn lint:js
+        - yarn lint:markdown
+        - yarn lint:social
+        - yarn build
+        - yarn lint:links
+
+    - stage: Build
+      name: Proselint
+      language: python
+      python: 3.6
+      cache:
+        pip: true
+        directories:
+          - $HOME/.cache
+      install: pip install -r requirements.txt
+      script: cp .proselintrc ~/ && proselint src/content
+
+    - stage: Post-build
+      name: Deploy
+      if: branch = master
+      script: bash ./src/scripts/deploy.sh
+
+    - stage: Post-build
+      name: External Link Check
+      script: yarn linkcheck

package.json

@@ -26,17 +26,24 @@
   "scripts": {
     "clean": "rimraf ./dist src/content/**/_*.md src/**/_*.json",
     "start": "cross-env NODE_ENV=development webpack-dev-server --config webpack.dev.js --env.dev",
+    "update-repos": "node src/utilities/fetch-package-repos.js",
     "content": "node src/scripts/build-content-tree.js ./src/content ./src/_content.json",
     "build-test": "npm run build && http-server build/",
-    "fetch": "sh src/scripts/fetch.sh",
+    "fetch": "run-p fetch:*",
+    "fetch:readmes": "node src/utilities/fetch-package-readmes.js",
+    "fetch:supporters": "node src/utilities/fetch-supporters.js",
+    "fetch:starter-kits": "node src/utilities/fetch-starter-kits.js",
     "prebuild": "npm run clean",
-    "build": "npm run fetch && npm run content && cross-env NODE_ENV=production webpack --config webpack.prod.js && npm run sitemap && echo webpack.js.org > dist/CNAME",
+    "build": "run-s fetch content && cross-env NODE_ENV=production webpack --config webpack.prod.js",
+    "postbuild": "npm run sitemap",
     "test": "npm run lint",
     "lint": "run-s lint:*",
-    "lint:js": "eslint src --ext .js,.jsx,.md",
+    "lint:js": "eslint src --ext .js,.jsx,.md --cache true --cache-location .cache/.eslintcache",
     "lint:markdown": "markdownlint --config ./.markdownlint.json *.md ./src/content/**/*.md --ignore './src/content/**/_*.md'",
     "lint:social": "alex . -q",
     "lint:prose": "cp .proselintrc ~/ && proselint src/content",
+    "lint:links": "hyperlink -r dist/index.html --canonicalroot https://webpack.js.org/ -i | tee internal-links.tap | tap-spot",
+    "linkcheck": "hyperlink -r dist/index.html --canonicalroot https://webpack.js.org/ --skip support__ --skip sidecar.gitter.im --skip vimdoc.sourceforge.net --skip img.shields.io --skip npmjs.com/package/ --skip opencollective.com/webpack --todo external-redirect | tee external-links.tap | tap-spot",
     "sitemap": "cd dist && sitemap-static --prefix=https://webpack.js.org/ > sitemap.xml",
     "serve": "npm run build && sirv start ./dist --port 4000",
     "deploy": "gh-pages -d dist"
@@ -47,11 +54,15 @@
     }
   },
   "lint-staged": {
-    "*.{js,jsx,md}": ["npm run lint:js"],
-    "*.md": ["npm run lint:markdown"]
+    "*.{js,jsx,md}": [
+      "npm run lint:js"
+    ],
+    "*.md": [
+      "npm run lint:markdown"
+    ]
   },
   "devDependencies": {
-    "@octokit/rest": "^15.2.6",
+    "@octokit/rest": "^15.9.4",
     "alex": "^5.1.0",
     "autoprefixer": "^7.2.3",
     "babel-core": "^6.26.0",
@@ -62,7 +73,7 @@
     "babel-plugin-transform-object-rest-spread": "^6.26.0",
     "babel-preset-env": "^1.6.0",
     "babel-preset-react": "^6.24.1",
-    "copy-webpack-plugin": "^4.3.0",
+    "copy-webpack-plugin": "4.5.2",
     "cross-env": "5.2.0",
     "css-loader": "^0.28.5",
     "directory-tree": "2.1.0",
@@ -81,7 +92,7 @@
     "html-webpack-template": "^6.1.0",
     "http-server": "^0.10.0",
     "husky": "^1.0.0-rc.8",
-    "hyperlink": "^3.0.1",
+    "hyperlink": "^4.0.0",
     "lint-staged": "^7.2.0",
     "loader-utils": "^1.1.0",
     "lodash": "^4.17.4",
@@ -113,7 +124,7 @@
     "sitemap-static": "^0.4.2",
     "static-site-generator-webpack-plugin": "^3.4.1",
     "style-loader": "^0.18.2",
-    "tap-parser": "^6.0.1",
+    "tap-spot": "^1.1.1",
     "through2": "^2.0.3",
     "uglifyjs-webpack-plugin": "^1.1.6",
     "webpack": "^3.10.0",
@@ -122,9 +133,9 @@
   },
   "dependencies": {
     "@rigor789/remark-autolink-headings": "^5.1.0",
+    "ajv": "^5.5.2",
     "docsearch.js": "^2.5.2",
     "gitter-sidecar": "^1.2.3",
-    "ajv": "^5.5.2",
     "javascriptstuff-db": "^1.12.0",
     "lodash.throttle": "^4.1.1",
     "prop-types": "^15.5.10",

repositories/README.md

@@ -0,0 +1,3 @@
+# Webpack documentation repositories
+
+The files in this directory are auto generated from `src/utils/fetch-package-repos.js` and should not be edited by hand. Any manual changes will be overwritten by the automation next time it runs.

repositories/loaders.json

@@ -0,0 +1,45 @@
+[
+  "webpack-contrib/json-loader",
+  "webpack-contrib/raw-loader",
+  "webpack-contrib/coffee-loader",
+  "webpack-contrib/css-loader",
+  "webpack-contrib/style-loader",
+  "webpack-contrib/script-loader",
+  "webpack-contrib/less-loader",
+  "webpack-contrib/bundle-loader",
+  "webpack-contrib/val-loader",
+  "webpack-contrib/file-loader",
+  "webpack-contrib/url-loader",
+  "webpack-contrib/i18n-loader",
+  "webpack-contrib/json5-loader",
+  "webpack-contrib/worker-loader",
+  "webpack-contrib/jshint-loader",
+  "webpack-contrib/imports-loader",
+  "webpack-contrib/exports-loader",
+  "webpack-contrib/mocha-loader",
+  "webpack-contrib/coverjs-loader",
+  "webpack-contrib/expose-loader",
+  "webpack-contrib/node-loader",
+  "webpack-contrib/coffee-redux-loader",
+  "webpack-contrib/transform-loader",
+  "webpack-contrib/html-loader",
+  "webpack-contrib/sass-loader",
+  "webpack-contrib/source-map-loader",
+  "webpack-contrib/react-proxy-loader",
+  "webpack-contrib/null-loader",
+  "webpack-contrib/multi-loader",
+  "webpack-contrib/istanbul-instrumenter-loader",
+  "webpack-contrib/eslint-loader",
+  "webpack-contrib/yaml-frontmatter-loader",
+  "webpack-contrib/svg-inline-loader",
+  "webpack-contrib/restyle-loader",
+  "webpack-contrib/gzip-loader",
+  "webpack-contrib/cache-loader",
+  "webpack-contrib/thread-loader",
+  "webpack-contrib/polymer-webpack-loader",
+  "webpack-contrib/workerize-loader",
+  "webpack-contrib/config-loader",
+  "babel/babel-loader",
+  "postcss/postcss-loader",
+  "peerigon/extract-loader"
+]

repositories/plugins.json

@@ -0,0 +1,15 @@
+[
+  "webpack-contrib/i18n-webpack-plugin",
+  "webpack-contrib/component-webpack-plugin",
+  "webpack-contrib/compression-webpack-plugin",
+  "webpack-contrib/extract-text-webpack-plugin",
+  "webpack-contrib/copy-webpack-plugin",
+  "webpack-contrib/npm-install-webpack-plugin",
+  "webpack-contrib/stylelint-webpack-plugin",
+  "webpack-contrib/babel-minify-webpack-plugin",
+  "webpack-contrib/uglifyjs-webpack-plugin",
+  "webpack-contrib/zopfli-webpack-plugin",
+  "webpack-contrib/closure-webpack-plugin",
+  "webpack-contrib/css-webpack-plugin",
+  "webpack-contrib/mini-css-extract-plugin"
+]

requirements.txt

@@ -0,0 +1 @@
+proselint

src/CNAME

@@ -0,0 +1 @@
+webpack.js.org

src/components/Link/Link.jsx

@@ -11,7 +11,7 @@ export default ({
   if ( url ) to = url;
 
   if ( to.startsWith('http') || to.startsWith('//') )
-    return <a href={ to } target="_blank" { ...props } />;
+    return <a href={ to } target="_blank" rel="noopener" { ...props } />;
 
   else return <Link to={ to } { ...props } />;
 };

src/components/Markdown/Markdown.scss

@@ -26,7 +26,7 @@
       color: inherit;
     }
 
-    a {
+    a[aria-hidden="true"] {
       margin-left: 8px;
       font-size: 0.8em;
       height: 1em;
@@ -35,7 +35,7 @@
       transition: all 250ms;
     }
 
-    &:hover a {
+    &:hover a[aria-hidden="true"] {
       opacity: 1;
       visibility: visible;
     }

src/components/Navigation/Navigation.jsx

@@ -46,7 +46,7 @@ export default class Navigation extends React.Component {
                 className="navigation__languages"
                 items={[
                   { title: 'English', url: 'https://webpack.js.org/' },
-                  { lang: 'zh', title: '中文', url: 'https://doc.webpack-china.org/' }
+                  { lang: 'zh', title: '中文', url: 'https://webpack.docschina.org/' }
                 ]} />
             )
           }

src/components/Organization/Organization.jsx

@@ -8,7 +8,7 @@ import './Organization.scss';
 
 const Organization = props => {
   return (
-    <Container className="organization page__content">
+    <Container className="organization page__content markdown">
       <h1>The Organization</h1>
 
       <p>The list below provides a brief overview of all commonly used projects in the webpack ecosystem.</p>

src/components/Organization/Organization.scss

@@ -3,6 +3,10 @@
 
 .organization {
   padding: 1.5em;
+
+  &__title > h4 {
+    margin-top: 0; // Overrides markdown scss
+  }
 }
 
 .organization__projects {