Provide an 'auto' option for --line_terminator

If 'auto' is chosen, output CRLF line endings if more than 50% of the
input is CRLF, otherwise LF.

The other choicese, CRLF or LF, behave as before.

NB: The default changes with this change: previously, we always
converted a CRLF input to LF, now the default is 'auto'

Author: hzeller

verible/common/formatting/basic-format-style-init.cc

@@ -37,8 +37,12 @@ ABSL_FLAG(int, over_column_limit_penalty, 100,
 ABSL_FLAG(int, line_break_penalty, 2,
           "Penalty added to solution for each introduced line break.");
 
-ABSL_FLAG(verible::LineTerminatorStyle, line_terminator,
-          verible::LineTerminatorStyle::kLF, "Line terminator");
+ABSL_FLAG(verible::LineTerminatorOptionStyle, line_terminator,
+          verible::LineTerminatorOptionStyle::kAuto,
+          "Line terminator. "
+          "The 'auto' option chooses the output depending on the observed "
+          "input. The explicit choice CR or CRLF fixes the output line "
+          "terminator.");
 
 namespace verible {
 void InitializeFromFlags(BasicFormatStyle *style) {

verible/common/formatting/basic-format-style.cc

@@ -48,14 +48,30 @@ std::string AbslUnparseFlag(const IndentationStyle &mode) {
   return stream.str();
 }
 
-static const verible::EnumNameMap<LineTerminatorStyle> &
-LineTerminatorStyleStrings() {
-  static const verible::EnumNameMap<LineTerminatorStyle>
-      kLineTerminatorStyleStringMap({
-          {"CRLF", LineTerminatorStyle::kCRLF},
-          {"LF", LineTerminatorStyle::kLF},
+static const verible::EnumNameMap<LineTerminatorOptionStyle> &
+LineTerminatorOptionStyleStrings() {
+  static const verible::EnumNameMap<LineTerminatorOptionStyle>
+      kLineTerminatorOptionStyleStringMap({
+          {"CRLF", LineTerminatorOptionStyle::kCRLF},
+          {"LF", LineTerminatorOptionStyle::kLF},
+          {"auto", LineTerminatorOptionStyle::kAuto},
       });
-  return kLineTerminatorStyleStringMap;
+  return kLineTerminatorOptionStyleStringMap;
+}
+
+std::ostream &operator<<(std::ostream &stream,
+                         LineTerminatorOptionStyle style) {
+  return LineTerminatorOptionStyleStrings().Unparse(style, stream);
+}
+
+bool AbslParseFlag(std::string_view text, LineTerminatorOptionStyle *mode,
+                   std::string *error) {
+  return LineTerminatorOptionStyleStrings().Parse(text, mode, error,
+                                                  "LineTerminatorOptionStyle");
+}
+
+std::string AbslUnparseFlag(const LineTerminatorOptionStyle &mode) {
+  return std::string{LineTerminatorOptionStyleStrings().EnumName(mode)};
 }
 
 void EmitLineTerminator(LineTerminatorStyle style, std::ostream &stream) {
@@ -69,18 +85,4 @@ void EmitLineTerminator(LineTerminatorStyle style, std::ostream &stream) {
   }
 }
 
-std::ostream &operator<<(std::ostream &stream, LineTerminatorStyle style) {
-  return LineTerminatorStyleStrings().Unparse(style, stream);
-}
-
-bool AbslParseFlag(std::string_view text, LineTerminatorStyle *mode,
-                   std::string *error) {
-  return LineTerminatorStyleStrings().Parse(text, mode, error,
-                                            "LineTerminatorStyle");
-}
-
-std::string AbslUnparseFlag(const LineTerminatorStyle &mode) {
-  return std::string{LineTerminatorStyleStrings().EnumName(mode)};
-}
-
 }  // namespace verible

verible/common/formatting/basic-format-style.h

@@ -21,20 +21,20 @@
 
 namespace verible {
 
-enum class LineTerminatorStyle {
-  // Line Feed `\n` (UNIX Style)
+// The option style allows for 'auto' which then is converted to the observed
+// style from the input (into a LineTerminatorStyle).
+enum class LineTerminatorOptionStyle {
   kLF,
-  // Carriage return + Line Feed `\r\n` (DOS Style)
   kCRLF,
+  kAuto,
 };
 
-void EmitLineTerminator(LineTerminatorStyle style, std::ostream &stream);
+std::ostream &operator<<(std::ostream &stream, LineTerminatorOptionStyle style);
 
-std::ostream &operator<<(std::ostream &stream, LineTerminatorStyle style);
+bool AbslParseFlag(std::string_view, LineTerminatorOptionStyle *,
+                   std::string *);
 
-bool AbslParseFlag(std::string_view, LineTerminatorStyle *, std::string *);
-
-std::string AbslUnparseFlag(const LineTerminatorStyle &);
+std::string AbslUnparseFlag(const LineTerminatorOptionStyle &);
 
 // Style configuration common to all languages.
 struct BasicFormatStyle {
@@ -57,8 +57,9 @@ struct BasicFormatStyle {
   // Penalty added to solution for each introduced line break.
   int line_break_penalty = 2;
 
-  // Line terminator character sequence
-  LineTerminatorStyle line_terminator = LineTerminatorStyle::kLF;
+  // Line terminator character sequence. Consistent LF for unit tests, but
+  // note, the command line flag sets this to 'auto'.
+  LineTerminatorOptionStyle line_terminator = LineTerminatorOptionStyle::kLF;
 
   // -- Note: when adding new fields, add them in basic_format_style_init.cc
 };
@@ -81,6 +82,16 @@ bool AbslParseFlag(std::string_view, IndentationStyle *, std::string *);
 
 std::string AbslUnparseFlag(const IndentationStyle &);
 
+// Concrete terminator.
+enum class LineTerminatorStyle {
+  // Line Feed `\n` (UNIX Style)
+  kLF,
+  // Carriage return + Line Feed `\r\n` (DOS Style)
+  kCRLF,
+};
+
+void EmitLineTerminator(LineTerminatorStyle style, std::ostream &stream);
+
 }  // namespace verible
 
 #endif  // VERIBLE_COMMON_FORMATTING_BASIC_FORMAT_STYLE_H_

verible/verilog/formatting/BUILD

@@ -138,6 +138,7 @@ cc_library(
         ":format-style",
         ":token-annotator",
         ":tree-unwrapper",
+        "//verible/common/formatting:basic-format-style",
         "//verible/common/formatting:format-token",
         "//verible/common/formatting:layout-optimizer",
         "//verible/common/formatting:line-wrap-searcher",

verible/verilog/formatting/formatter.cc

@@ -16,6 +16,7 @@
 
 #include <algorithm>
 #include <cstddef>
+#include <cstdint>
 #include <cstdlib>
 #include <functional>
 #include <iostream>
@@ -29,6 +30,7 @@
 #include "absl/status/status.h"
 #include "absl/status/statusor.h"
 #include "absl/strings/str_cat.h"
+#include "verible/common/formatting/basic-format-style.h"
 #include "verible/common/formatting/format-token.h"
 #include "verible/common/formatting/layout-optimizer.h"
 #include "verible/common/formatting/line-wrap-searcher.h"
@@ -970,6 +972,50 @@ Status Formatter::Format(const ExecutionControl &control) {
   return absl::OkStatus();
 }
 
+// This might maybe something in the verible/common/text
+struct TerminatorCount {
+  int64_t line_count = 0;
+  int64_t crlf_count = 0;
+};
+
+static TerminatorCount CountLineTerminators(std::string_view text,
+                                            int64_t count_at_most) {
+  // TODO: maybe it is sufficient to just look at the first 1000 lines or so.
+  TerminatorCount result;
+  size_t pos = 0;
+  while ((pos = text.find_first_of('\n', pos)) != std::string_view::npos) {
+    ++result.line_count;
+    if (pos > 0 && text[pos - 1] == '\r') {
+      ++result.crlf_count;
+    }
+    ++pos;
+    if (result.line_count >= count_at_most) {
+      break;
+    }
+  }
+  return result;
+}
+
+// From options, extraact the line terminator style. If 'auto' was chosen,
+// attempt to determine from text.
+static verible::LineTerminatorStyle DetermineOutputLineTerminator(
+    verible::LineTerminatorOptionStyle from_options, std::string_view text) {
+  static constexpr int64_t kCountAtMost = 100;  // sufficient stats
+  switch (from_options) {
+    case verible::LineTerminatorOptionStyle::kCRLF:
+      return verible::LineTerminatorStyle::kCRLF;
+    case verible::LineTerminatorOptionStyle::kLF:
+      return verible::LineTerminatorStyle::kLF;
+    case verible::LineTerminatorOptionStyle::kAuto: {
+      const TerminatorCount counts = CountLineTerminators(text, kCountAtMost);
+      return (counts.crlf_count >= counts.line_count / 2)
+                 ? verible::LineTerminatorStyle::kCRLF
+                 : verible::LineTerminatorStyle::kLF;
+    }
+  }
+  return verible::LineTerminatorStyle::kLF;
+}
+
 void Formatter::Emit(bool include_disabled, std::ostream &stream) const {
   const std::string_view full_text(text_structure_.Contents());
   std::function<bool(const verible::TokenInfo &)> include_token_p;
@@ -981,6 +1027,8 @@ void Formatter::Emit(bool include_disabled, std::ostream &stream) const {
     };
   }
 
+  const verible::LineTerminatorStyle out_terminator =
+      DetermineOutputLineTerminator(style_.line_terminator, full_text);
   int position = 0;  // tracks with the position in the original full_text
   for (const verible::FormattedExcerpt &line : formatted_lines_) {
     // TODO(fangism): The handling of preserved spaces before tokens is messy:
@@ -993,7 +1041,7 @@ void Formatter::Emit(bool include_disabled, std::ostream &stream) const {
         full_text.substr(position, front_offset - position));
     FormatWhitespaceWithDisabledByteRanges(full_text, leading_whitespace,
                                            disabled_ranges_, include_disabled,
-                                           stream, style_.line_terminator);
+                                           stream, out_terminator);
 
     // When front of first token is format-disabled, the previous call will
     // already cover the space up to the front token, in which case,
@@ -1010,7 +1058,7 @@ void Formatter::Emit(bool include_disabled, std::ostream &stream) const {
   const std::string_view trailing_whitespace(full_text.substr(position));
   FormatWhitespaceWithDisabledByteRanges(full_text, trailing_whitespace,
                                          disabled_ranges_, include_disabled,
-                                         stream, style_.line_terminator);
+                                         stream, out_terminator);
 }
 
 }  // namespace formatter

verible/verilog/tools/formatter/README.md

@@ -23,6 +23,9 @@ To pipe from stdin, use '-' as <file>.
       default: 2;
     --line_break_penalty (Penalty added to solution for each introduced line
       break.); default: 2;
+    --line_terminator (Line terminator. The 'auto' option chooses the output
+      depending on the observed input. The explicit choice CR or CRLF fixes the
+      output line terminator.); default: auto;
     --over_column_limit_penalty (For penalty minimization, this represents the
       baseline penalty value of exceeding the column limit. Additional penalty
       of 1 is incurred for each character over this limit); default: 100;

verible/verilog/tools/formatter/format_line_terminator_test.sh

@@ -37,16 +37,27 @@ done
 # Test any combination of input line terminators and output line terminators.
 # Test both inline formatting and standard output
 for original_newline in LF CRLF; do
+  PROPER_INPUT_FILE="${MY_INPUT_FILE}$original_newline"
+
+  # For 'auto', the line terminators of the output will be the same as input.
+  cp $PROPER_INPUT_FILE ${MY_OUTPUT_FILE}
+  ${formatter} --line_terminator=auto ${PROPER_INPUT_FILE} > ${MY_OUTPUT_FILE}
+  cmp ${MY_OUTPUT_FILE} $PROPER_INPUT_FILE || exit 1
+
+  cp $PROPER_INPUT_FILE ${MY_OUTPUT_FILE}
+  ${formatter} --line_terminator=auto --inplace ${MY_OUTPUT_FILE}
+  cmp ${MY_OUTPUT_FILE} $PROPER_INPUT_FILE || exit 2
+
+  # With an explicit target newline, we expect that particular one.
   for target_newline in LF CRLF; do
-    PROPER_INPUT_FILE="${MY_INPUT_FILE}$original_newline"
     PROPER_EXPECT_FILE="${MY_EXPECT_FILE}$target_newline"
 
     ${formatter} --line_terminator=$target_newline $PROPER_INPUT_FILE > ${MY_OUTPUT_FILE}
-    cmp ${MY_OUTPUT_FILE} $PROPER_EXPECT_FILE || exit 1
+    cmp ${MY_OUTPUT_FILE} $PROPER_EXPECT_FILE || exit 3
 
     cp $PROPER_INPUT_FILE ${MY_OUTPUT_FILE}
     ${formatter} --line_terminator=$target_newline --inplace ${MY_OUTPUT_FILE}
-    cmp ${MY_OUTPUT_FILE} $PROPER_EXPECT_FILE || exit 2
+    cmp ${MY_OUTPUT_FILE} $PROPER_EXPECT_FILE || exit 4
   done
 done