Add remove_duplicates ignore_comments and merge_comments logic

README.md

@@ -899,6 +899,11 @@ rotation: foo
 
 The duplicate handling can be changed with the switch `remove_duplicates`:
 
+*   `yes` (default): Deduplicates based on both code and comments.
+*   `no`: Leaves duplicates untouched.
+*   `keep_first_comment`: Deduplicates based on code lines only, retaining only the comment of the very first duplicate occurrence.
+*   `merge_comments`: Deduplicates based on code lines only, merging unique comments from all occurrences to the single remaining entry.
+
 ```diff
 +# keep-sorted start remove_duplicates=no
  rotation: bar

goldens/duplicates_keep_first_comment.in

@@ -0,0 +1,18 @@
+Remove duplicates but ignore comments:
+// keep-sorted-test start remove_duplicates=keep_first_comment sticky_comments=yes
+// First foo
+foo
+bar
+// Second foo
+foo
+// Third foo
+foo
+// keep-sorted-test end
+
+Remove duplicates ignoring comments, when the first item has no comment:
+// keep-sorted-test start remove_duplicates=keep_first_comment sticky_comments=yes
+foo
+bar
+// Second foo
+foo
+// keep-sorted-test end

goldens/duplicates_keep_first_comment.out

@@ -0,0 +1,12 @@
+Remove duplicates but ignore comments:
+// keep-sorted-test start remove_duplicates=keep_first_comment sticky_comments=yes
+bar
+// First foo
+foo
+// keep-sorted-test end
+
+Remove duplicates ignoring comments, when the first item has no comment:
+// keep-sorted-test start remove_duplicates=keep_first_comment sticky_comments=yes
+bar
+foo
+// keep-sorted-test end

goldens/duplicates_merge_comments.in

@@ -0,0 +1,21 @@
+Merge duplicate comments:
+// keep-sorted-test start remove_duplicates=merge_comments sticky_comments=yes
+// First foo
+foo
+bar
+// Second foo
+foo
+// Third foo
+foo
+// keep-sorted-test end
+
+Merge duplicate comments where some are identical:
+// keep-sorted-test start remove_duplicates=merge_comments sticky_comments=yes
+// Common foo flag
+foo
+bar
+// Common foo flag
+foo
+// Different foo flag
+foo
+// keep-sorted-test end

goldens/duplicates_merge_comments.out

@@ -0,0 +1,16 @@
+Merge duplicate comments:
+// keep-sorted-test start remove_duplicates=merge_comments sticky_comments=yes
+bar
+// First foo
+// Second foo
+// Third foo
+foo
+// keep-sorted-test end
+
+Merge duplicate comments where some are identical:
+// keep-sorted-test start remove_duplicates=merge_comments sticky_comments=yes
+bar
+// Common foo flag
+// Different foo flag
+foo
+// keep-sorted-test end

keepsorted/block.go

@@ -259,15 +259,36 @@ func (b block) sorted() (sorted []string, alreadySorted bool) {
 	}
 
 	removedDuplicate := false
-	if b.metadata.opts.RemoveDuplicates {
-		seen := map[string]bool{}
+	if b.metadata.opts.RemoveDuplicates != DuplicateResolutionFalse {
+		seenStrings := map[string]bool{}
+		seenLines := map[string]*lineGroup{}
 		var deduped []*lineGroup
 		for _, lg := range groups {
-			if s := lg.String(); !seen[s] {
-				seen[s] = true
-				deduped = append(deduped, lg)
+			if b.metadata.opts.RemoveDuplicates == DuplicateResolutionTrue {
+				if s := lg.String(); !seenStrings[s] {
+					seenStrings[s] = true
+					deduped = append(deduped, lg)
+				} else {
+					removedDuplicate = true
+				}
 			} else {
-				removedDuplicate = true
+				codeMapKey := strings.Join(lg.lines, "\n")
+
+				if firstLg, ok := seenLines[codeMapKey]; !ok {
+					seenLines[codeMapKey] = lg
+					deduped = append(deduped, lg)
+				} else {
+					removedDuplicate = true
+
+					if b.metadata.opts.RemoveDuplicates == DuplicateResolutionMergeComments {
+						firstLg.comment = slices.Clone(firstLg.comment)
+						for _, newComment := range lg.comment {
+							if !slices.Contains(firstLg.comment, newComment) {
+								firstLg.comment = append(firstLg.comment, newComment)
+							}
+						}
+					}
+				}
 			}
 		}
 		groups = deduped

keepsorted/keep_sorted_test.go

@@ -838,7 +838,7 @@ func TestLineSorting(t *testing.T) {
 			name: "AlreadySorted_ExceptForDuplicate",
 
 			opts: blockOptions{
-				RemoveDuplicates: true,
+				RemoveDuplicates: DuplicateResolutionTrue,
 			},
 			in: []string{
 				"Bar",
@@ -1020,7 +1020,7 @@ func TestLineSorting(t *testing.T) {
 
 			opts: func() blockOptions {
 				opts := blockOptions{
-					RemoveDuplicates: true,
+					RemoveDuplicates: DuplicateResolutionTrue,
 					StickyComments:   true,
 				}
 				opts.setCommentMarker("//")
@@ -1048,7 +1048,7 @@ func TestLineSorting(t *testing.T) {
 			name: "RemoveDuplicates_IgnoresTraliningCommas",
 
 			opts: blockOptions{
-				RemoveDuplicates: true,
+				RemoveDuplicates: DuplicateResolutionTrue,
 			},
 			in: []string{
 				"foo,",
@@ -1065,7 +1065,7 @@ func TestLineSorting(t *testing.T) {
 			name: "RemoveDuplicates_IgnoresTrailingCommas_RemovesCommaIfLastElement",
 
 			opts: blockOptions{
-				RemoveDuplicates: true,
+				RemoveDuplicates: DuplicateResolutionTrue,
 			},
 			in: []string{
 				"foo,",
@@ -1082,7 +1082,7 @@ func TestLineSorting(t *testing.T) {
 			name: "RemoveDuplicates_IgnoresTrailingCommas_RemovesCommaIfOnlyElement",
 
 			opts: blockOptions{
-				RemoveDuplicates: true,
+				RemoveDuplicates: DuplicateResolutionTrue,
 			},
 			in: []string{
 				"foo,",
@@ -1097,7 +1097,7 @@ func TestLineSorting(t *testing.T) {
 			name: "RemoveDuplicates_Keep",
 
 			opts: blockOptions{
-				RemoveDuplicates: false,
+				RemoveDuplicates: DuplicateResolutionFalse,
 			},
 			in: []string{
 				"foo",

keepsorted/options.go

@@ -35,6 +35,15 @@ import (
 // true is unmarshaled as 1, false as 0.
 type IntOrBool int
 
+type DuplicateResolution int
+
+const (
+	DuplicateResolutionFalse DuplicateResolution = iota
+	DuplicateResolutionTrue
+	DuplicateResolutionKeepFirstComment
+	DuplicateResolutionMergeComments
+)
+
 type ByRegexOption struct {
 	Pattern  *regexp.Regexp
 	Template *string
@@ -147,7 +156,7 @@ type blockOptions struct {
 	// Any other positive integer specifies the number of newlines to separate the groups.
 	NewlineSeparated IntOrBool `key:"newline_separated"`
 	// RemoveDuplicates determines whether we drop lines that are an exact duplicate.
-	RemoveDuplicates bool `key:"remove_duplicates"`
+	RemoveDuplicates DuplicateResolution `key:"remove_duplicates"`
 
 	// Syntax used to start a comment for keep-sorted annotation, e.g. "//".
 	commentMarker string
@@ -161,7 +170,7 @@ var (
 		StickyPrefixes:   nil, // Will be populated with the comment marker of the start directive.
 		Order:            OrderAsc,
 		CaseSensitive:    true,
-		RemoveDuplicates: true,
+		RemoveDuplicates: DuplicateResolutionTrue,
 	}
 
 	fieldIndexByKey map[string]int
@@ -250,6 +259,19 @@ func formatValue(val reflect.Value) (string, error) {
 		default:
 			return strconv.Itoa(i), nil
 		}
+	case reflect.TypeFor[DuplicateResolution]():
+		switch val.Interface().(DuplicateResolution) {
+		case DuplicateResolutionFalse:
+			return "no", nil
+		case DuplicateResolutionTrue:
+			return "yes", nil
+		case DuplicateResolutionKeepFirstComment:
+			return "keep_first_comment", nil
+		case DuplicateResolutionMergeComments:
+			return "merge_comments", nil
+		default:
+			panic(fmt.Errorf("unhandled DuplicateResolution value: %v", val))
+		}
 	case reflect.TypeFor[int]():
 		return strconv.Itoa(int(val.Int())), nil
 	case reflect.TypeFor[[]int]():

keepsorted/options_parser.go

@@ -59,6 +59,9 @@ func (p *parser) popValue(typ reflect.Type) (reflect.Value, error) {
 	case reflect.TypeFor[int]():
 		val, err := p.popInt()
 		return reflect.ValueOf(val), err
+	case reflect.TypeFor[DuplicateResolution]():
+		val, err := p.popDuplicateResolution()
+		return reflect.ValueOf(val), err
 	case reflect.TypeFor[[]int]():
 		val, err := p.popIntList()
 		return reflect.ValueOf(val), err
@@ -98,6 +101,23 @@ func (p *parser) popBool() (bool, error) {
 	return b, nil
 }
 
+func (p *parser) popDuplicateResolution() (DuplicateResolution, error) {
+	val, rest, _ := strings.Cut(p.line, " ")
+	p.line = rest
+	switch val {
+	case "yes", "true":
+		return DuplicateResolutionTrue, nil
+	case "no", "false":
+		return DuplicateResolutionFalse, nil
+	case "keep_first_comment":
+		return DuplicateResolutionKeepFirstComment, nil
+	case "merge_comments":
+		return DuplicateResolutionMergeComments, nil
+	default:
+		return DuplicateResolutionFalse, fmt.Errorf("unrecognized remove_duplicates value %q", val)
+	}
+}
+
 func (p *parser) popInt() (int, error) {
 	val, rest, _ := strings.Cut(p.line, " ")
 	p.line = rest

keepsorted/options_parser_test.go

@@ -284,6 +284,32 @@ func TestPopValue(t *testing.T) {
 			want:    IntOrBool(0),
 			wantErr: true,
 		},
+		{
+			name:  "DuplicateResolution_True",
+			input: "yes",
+			want:  DuplicateResolutionTrue,
+		},
+		{
+			name:  "DuplicateResolution_False",
+			input: "no",
+			want:  DuplicateResolutionFalse,
+		},
+		{
+			name:  "DuplicateResolution_KeepFirstComment",
+			input: "keep_first_comment",
+			want:  DuplicateResolutionKeepFirstComment,
+		},
+		{
+			name:  "DuplicateResolution_MergeComments",
+			input: "merge_comments",
+			want:  DuplicateResolutionMergeComments,
+		},
+		{
+			name:    "DuplicateResolution_Invalid",
+			input:   "foo",
+			want:    DuplicateResolutionFalse,
+			wantErr: true,
+		},
 	} {
 		t.Run(tc.name, func(t *testing.T) {
 			suffix := "trailing content..."

keepsorted/options_test.go

@@ -290,6 +290,31 @@ func TestBlockOptions(t *testing.T) {
 				GroupStartRegex: []*regexp.Regexp{regexp.MustCompile("^CREATE"), regexp.MustCompile("b")},
 			},
 		},
+		{
+			name: "RemoveDuplicates_Yes",
+			in:   "remove_duplicates=yes",
+			want: blockOptions{RemoveDuplicates: DuplicateResolutionTrue},
+		},
+		{
+			name: "RemoveDuplicates_No",
+			in:   "remove_duplicates=no",
+			want: blockOptions{RemoveDuplicates: DuplicateResolutionFalse},
+		},
+		{
+			name: "RemoveDuplicates_KeepFirstComment",
+			in:   "remove_duplicates=keep_first_comment",
+			want: blockOptions{RemoveDuplicates: DuplicateResolutionKeepFirstComment},
+		},
+		{
+			name: "RemoveDuplicates_MergeComments",
+			in:   "remove_duplicates=merge_comments",
+			want: blockOptions{RemoveDuplicates: DuplicateResolutionMergeComments},
+		},
+		{
+			name:    "RemoveDuplicates_Invalid",
+			in:      "remove_duplicates=nah",
+			wantErr: `while parsing option "remove_duplicates": unrecognized remove_duplicates value "nah"`,
+		},
 	} {
 		t.Run(tc.name, func(t *testing.T) {
 			initZerolog(t)