detect and normalize line endings. (#19501)

jhipster · Aug 24, 2022 · 98bcfd3 · 98bcfd3
1 parent a6f6dd4
commit 98bcfd3
Show file tree

Hide file tree

Showing 10 changed files with 146 additions and 5 deletions.
diff --git a/.gitattributes b/.gitattributes
@@ -1,5 +1,8 @@
 # All text files should have the "lf" (Unix) line endings
 * text eol=lf
+*.cmd text eol=crlf
+*.cmd.ejs text eol=crlf
+*.bat text eol=crlf
 
 # Explicitly declare text files you want to always be normalized and converted
 # to native line endings on checkout.

diff --git a/.github/dependabot.yml b/.github/dependabot.yml
@@ -20,6 +20,8 @@ updates:
         versions: ['>=5']
       - dependency-name: 'escape-string-regexp'
         versions: ['>=5']
+      - dependency-name: 'isbinaryfile'
+        versions: ['>=5']
 
   - package-ecosystem: 'npm'
     directory: '/generators/client/templates/angular/'

diff --git a/generators/app/index.js b/generators/app/index.js
@@ -301,6 +301,11 @@ module.exports = class JHipsterAppGenerator extends BaseBlueprintGenerator {
       type: Boolean,
     });
 
+    this.option('auto-crlf', {
+      desc: 'Detect line endings',
+      type: Boolean,
+    });
+
     // Just constructing help, stop here
     if (this.options.help) {
       return;

diff --git a/generators/bootstrap/index.js b/generators/bootstrap/index.js
@@ -25,6 +25,9 @@ const {
   patternFilter,
   patternSpy,
 } = require('yeoman-environment/transform');
+const { transform } = require('p-transform');
+const { stat } = require('fs/promises');
+const { isBinaryFile } = require('isbinaryfile');
 
 const { hasState, setModifiedFileState } = State;
 
@@ -44,6 +47,7 @@ const { prepareFieldForTemplates } = require('../../utils/field');
 const { createUserEntity } = require('../../utils/user');
 const { OAUTH2 } = require('../../jdl/jhipster/authentication-types');
 const { CommonDBTypes } = require('../../jdl/jhipster/field-types');
+const { detectCrLf, normalizeLineEndings } = require('../utils');
 
 const { LONG: TYPE_LONG } = CommonDBTypes;
 
@@ -161,7 +165,7 @@ module.exports = class extends BaseGenerator {
    */
   async _commitSharedFs(stream = this.env.sharedFs.stream(), skipPrettier = this.options.skipPrettier) {
     const { skipYoResolve } = this.options;
-    const { withGeneratedFlag } = this.jhipsterConfig;
+    const { withGeneratedFlag, autoCrlf } = this.jhipsterConfig;
 
     // JDL writes directly to disk, set the file as modified so prettier will be applied
     const { upgradeCommand, ignoreErrors } = this.options;
@@ -199,6 +203,33 @@ module.exports = class extends BaseGenerator {
         file.conflicter = 'force';
       }, '**/.jhipster/*.json').name('jhipster:config-files:force');
 
+    const convertToCRLF = () =>
+      transform(async file => {
+        if (!file.contents) {
+          return file;
+        }
+        if (await isBinaryFile(file.contents)) {
+          return file;
+        }
+        const fstat = await stat(file.path);
+        if (!fstat.isFile()) {
+          return file;
+        }
+        const attributes = Object.fromEntries(
+          (await this.createGit().raw('check-attr', 'binary', 'eol', '--', file.path))
+            .split(/\r\n|\r|\n/)
+            .map(attr => attr.split(':'))
+            .map(([_file, attr, value]) => [attr, value])
+        );
+        if (attributes.binary === 'set' || attributes.eol !== 'lf') {
+          return file;
+        }
+        if (attributes.eol === 'crlf' || (await detectCrLf(file.path))) {
+          file.contents = Buffer.from(normalizeLineEndings(file.contents.toString(), '\r\n'));
+        }
+        return file;
+      }, 'jhipster:crlf');
+
     const transformStreams = [
       // multi-step changes the file path, should be executed earlier in the pipeline
       new MultiStepTransform(),
@@ -207,6 +238,7 @@ module.exports = class extends BaseGenerator {
       createForceWriteConfigFiles(),
       ...(withGeneratedFlag ? [generatedAnnotationTransform(this)] : []),
       ...(skipPrettier ? [] : [createApplyPrettierTransform()]),
+      ...(autoCrlf ? [convertToCRLF()] : []),
       createConflicterCheckTransform(this.env.conflicter, conflicterStatus),
       createConflicterStatusTransform(),
     ];

diff --git a/generators/generator-base.js b/generators/generator-base.js
@@ -2806,6 +2806,9 @@ templates: ${JSON.stringify(existingTemplates, null, 2)}`;
     if (options.enableTranslation !== undefined) {
       this.jhipsterConfig.enableTranslation = options.enableTranslation;
     }
+    if (options.autoCrlf !== undefined) {
+      this.jhipsterConfig.autoCrlf = options.autoCrlf;
+    }
     if (options.language) {
       // workaround double options parsing, remove once generator supports skipping parse options
       const languages = options.language.flat();

diff --git a/generators/utils.js b/generators/utils.js
@@ -32,6 +32,8 @@ const FileUtils = require('../jdl/utils/file-utils');
 const LANGUAGES_MAIN_SRC_DIR = `${__dirname}/languages/templates/${constants.CLIENT_MAIN_SRC_DIR}`;
 
 module.exports = {
+  detectCrLf,
+  normalizeLineEndings,
   rewrite,
   rewriteFile,
   replaceContent,
@@ -131,11 +133,49 @@ function escapeRegExp(str) {
  * @param {string} str string
  * @returns {string} string where CRLF is replaced with LF in Windows
  */
-function normalizeLineEndings(str) {
+function normalizeWindowsLineEndings(str) {
   const isWin32 = os.platform() === 'win32';
   return isWin32 ? str.replace(/\r\n/g, '\n') : str;
 }
 
+/**
+ * Replace line endings with the specified one.
+ *
+ * @param {string} str
+ * @param {string} lineEnding
+ * @returns {string} normalized line ending string
+ */
+function normalizeLineEndings(str, lineEnding) {
+  return str.replace(/\r\n|\r|\n/g, lineEnding);
+}
+
+/**
+ * Detect the file first line endings
+ *
+ * @param {string} filePath
+ * @returns {boolean|undefined} true in case of crlf, false in case of lf, undefined for a single line file
+ */
+function detectCrLf(filePath) {
+  return new Promise((resolve, reject) => {
+    let isCrlf;
+    const rs = fs.createReadStream(filePath, { encoding: 'utf8' });
+    rs.on('data', function (chunk) {
+      const n = chunk.indexOf('\n');
+      const r = chunk.indexOf('\r');
+      if (n !== -1 || r !== -1) {
+        isCrlf = n === -1 || (r !== -1 && r < n);
+        rs.close();
+      }
+    })
+      .on('close', function () {
+        resolve(isCrlf);
+      })
+      .on('error', function (err) {
+        reject(err);
+      });
+  });
+}
+
 /**
  * Change spaces sequences and '>' to allow any number of spaces or new line prefix
  *
@@ -167,7 +207,7 @@ function rewrite(args) {
       re = escapeRegExp(re);
     }
   } else {
-    re = args.splicable.map(line => `\\s*${escapeRegExp(normalizeLineEndings(line))}`).join('\n');
+    re = args.splicable.map(line => `\\s*${escapeRegExp(normalizeWindowsLineEndings(line))}`).join('\n');
   }
   if (!re.test) {
     if (args.prettierAware) {
@@ -176,7 +216,7 @@ function rewrite(args) {
     re = new RegExp(re);
   }
 
-  if (re.test(normalizeLineEndings(args.haystack))) {
+  if (re.test(normalizeWindowsLineEndings(args.haystack))) {
     return args.haystack;
   }
 

diff --git a/lib/support/base.cjs b/lib/support/base.cjs
@@ -17,6 +17,7 @@
  * limitations under the License.
  */
 const { platform } = require('os');
+const { normalizeLineEndings } = require('../../generators/utils');
 
 const isWin32 = platform() === 'win32';
 
@@ -29,7 +30,7 @@ const isWin32 = platform() === 'win32';
 const joinCallbacks = (...callbacks) => {
   return function (content, filePath) {
     if (isWin32 && content.match(/\r\n/)) {
-      callbacks = [ct => ct.replace(/\r\n/g, '\n')].concat(callbacks).concat(ct => ct.replace(/\n/g, '\r\n'));
+      callbacks = [ct => normalizeLineEndings(ct, '\n')].concat(callbacks).concat(ct => normalizeLineEndings(ct, '\r\n'));
     }
     for (const callback of callbacks) {
       content = callback.call(this, content, filePath);

diff --git a/package-lock.json b/package-lock.json
diff --git a/package.json b/package.json
@@ -96,6 +96,7 @@
     "escape-string-regexp": "4.0.0",
     "glob": "8.0.3",
     "insight": "0.11.1",
+    "isbinaryfile": "4.0.10",
     "js-yaml": "4.1.0",
     "lodash": "4.17.21",
     "mem-fs-editor": "9.5.0",

diff --git a/test/utils.spec.js b/test/utils.spec.js
@@ -1,9 +1,14 @@
 const assert = require('yeoman-assert');
 const path = require('path');
 const sinon = require('sinon');
+const { expect } = require('expect');
+const { writeFileSync } = require('fs');
+
 const utils = require('../generators/utils');
 const { prepareTempDir } = require('./utils/utils');
 
+const { detectCrLf } = utils;
+
 describe('JHipster Utils', () => {
   describe('::getJavadoc', () => {
     describe('when passing a negative or nil increment', () => {
@@ -364,4 +369,52 @@ describe('JHipster Utils', () => {
       );
     });
   });
+  describe('::detectCrLf', () => {
+    describe('passing a crlf file', () => {
+      let cleanup;
+      before(() => {
+        cleanup = prepareTempDir();
+        writeFileSync('crlf.txt', 'a\r\ncrlf file');
+      });
+      after(() => cleanup());
+
+      it('should return true', async () => {
+        expect(await detectCrLf('crlf.txt')).toBe(true);
+      });
+    });
+    describe('passing a lf file', () => {
+      let cleanup;
+      before(() => {
+        cleanup = prepareTempDir();
+        writeFileSync('lf.txt', 'a\nlf file');
+      });
+      after(() => cleanup());
+
+      it('should return false', async () => {
+        expect(await detectCrLf('lf.txt')).toBe(false);
+      });
+    });
+    describe('passing a single line file', () => {
+      let cleanup;
+      before(() => {
+        cleanup = prepareTempDir();
+        writeFileSync('lf.txt', 'a single line file');
+      });
+      after(() => cleanup());
+
+      it('should return undefined', async () => {
+        expect(await detectCrLf('lf.txt')).toBe(undefined);
+      });
+    });
+  });
+  describe('::normalizeLineEndings', () => {
+    it('should convert \\r\\n to \\n', () => {
+      expect(utils.normalizeLineEndings('a\r\ncrlf\r\nfile\r\nwith\nlf\nlines\r\n', '\r\n')).toBe(
+        'a\r\ncrlf\r\nfile\r\nwith\r\nlf\r\nlines\r\n'
+      );
+    });
+    it('should convert \\n to \\r\\n', () => {
+      expect(utils.normalizeLineEndings('a\r\ncrlf\r\nfile\r\nwith\nlf\nlines\r\n', '\n')).toBe('a\ncrlf\nfile\nwith\nlf\nlines\n');
+    });
+  });
 });