File Coverage

File:lib/CheckSpelling/SpellingCollator.pm
Coverage:86.1%

linestmtbrancondsubtimecode
1#! -*-perl-*-
2
3package CheckSpelling::SpellingCollator;
4
5our $VERSION='0.1.0';
6
1
1
1
107299
2
use 5.022;
7
1
1
1
1
2
2
40
use feature 'unicode_strings';
8
1
1
1
1
1
1
17
use warnings;
9
1
1
1
1
1
1
20
use File::Path qw(remove_tree);
10
1
1
1
1
230
1
2843
use CheckSpelling::Util;
11
12my %letter_map;
13my %ignored_event_map;
14my $disable_word_collating;
15
16my %last_seen;
17
18sub get_field {
19
44
44
31
  my ($record, $field) = @_;
20
44
100
391
  return 0 unless $record =~ (/\b$field:\s*(\d+)/);
21
32
38
  return $1;
22}
23
24sub get_array {
25
2
2
1
  my ($record, $field) = @_;
26
2
50
16
  return () unless $record =~ (/\b$field: \[([^\]]+)\]/);
27
2
2
  my $values = $1;
28
2
3
  return split /\s*,\s*/, $values;
29}
30
31sub maybe {
32
7
7
5
  my ($next, $value) = @_;
33
7
50
67
10
  $next = $value unless $next && $next < $value;
34
7
3
  return $next;
35}
36
37my %expected = ();
38sub expect_item {
39
104
104
63
  my ($item, $value) = @_;
40
104
35
  our %expected;
41
104
43
  my $next;
42
104
100
100
107
  if (defined $expected{$item}) {
43
26
28
    $next = $expected{$item};
44
26
100
19
    $next = $value if $value < $next;
45  } elsif ($item =~ /^([A-Z])(.*)/) {
46
12
8
    $item = $1 . lc $2;
47
12
100
8
    if (defined $expected{$item}) {
48
2
2
      $next = $expected{$item};
49
2
1
      $next = maybe($next, $value + .1);
50    } else {
51
10
8
      $item = lc $item;
52
10
100
5
      if (defined $expected{$item}) {
53
5
4
        $next = $expected{$item};
54
5
3
        $next = maybe($next, $value + .2);
55      }
56    }
57  }
58
104
100
92
  return 0 unless defined $next;
59
33
20
  $expected{$item} = $next;
60
33
71
  return $value;
61}
62
63sub skip_item {
64
55
55
28
  my ($word) = @_;
65
55
100
29
  return 1 if expect_item($word, 1);
66
35
21
  my $key = lc $word;
67
35
50
30
  return 2 if expect_item($key, 2);
68
35
100
100
100
43
  if ($key =~ /.s$/) {
69
2
100
4
    if ($key =~ /ies$/) {
70
1
1
      $key =~ s/ies$/y/;
71    } else {
72
1
1
      $key =~ s/s$//;
73    }
74  } elsif ($key =~ /^(.+[^aeiou])ed$/) {
75
1
1
    $key = $1;
76  } elsif ($key =~ /^(.+)'[ds]$/) {
77
6
6
    $key = $1;
78  } else {
79
26
28
    return 0;
80  }
81
9
50
3
  return 3 if expect_item($key, 3);
82
0
0
  return 0;
83}
84
85sub should_skip_warning {
86
76
76
43
  my ($warning) = @_;
87
76
100
97
  if ($warning =~ /\(([-\w]+)\)$/) {
88
72
49
    my ($code) = ($1);
89
72
22
    our %ignored_event_map;
90
72
100
55
    return 1 if $ignored_event_map{$code};
91  }
92
75
55
  return 0;
93}
94
95sub log_skip_item {
96
51
51
64
  my ($item, $file, $warning, $unknown_word_limit) = @_;
97
51
50
33
  return 1 if should_skip_warning $warning;
98
51
100
31
  return 1 if skip_item($item);
99
22
4
  our %seen;
100
22
33
  my $seen_count = $seen{$item};
101
22
100
13
  if (defined $seen_count) {
102
9
100
100
13
    if (!defined $unknown_word_limit || ($seen_count++ < $unknown_word_limit)) {
103
8
30
      print MORE_WARNINGS "$file$warning\n";
104    } else {
105
1
0
      our %last_seen;
106
1
2
      $last_seen{$item} = "$file$warning";
107    }
108
9
10
    $seen{$item} = $seen_count;
109
9
15
    return 1;
110  }
111
13
9
  $seen{$item} = 1;
112
13
12
  return 0;
113}
114
115sub stem_word {
116
22
22
12
  my ($key) = @_;
117
22
8
  our $disable_word_collating;
118
22
50
10
  return $key if $disable_word_collating;
119
120
22
100
100
23
  if ($key =~ /.s$/) {
121
3
100
4
    if ($key =~ /ies$/) {
122
1
2
      $key =~ s/ies$/y/;
123    } else {
124
2
2
      $key =~ s/s$//;
125    }
126  } elsif ($key =~ /.[^aeiou]ed$/) {
127
1
14
    $key =~ s/ed$//;
128  }
129
22
15
  return $key;
130}
131
132sub collate_key {
133
89
89
49
  my ($key) = @_;
134
89
41
  our $disable_word_collating;
135
89
35
  my $char;
136
89
100
49
  if ($disable_word_collating) {
137
16
16
    $char = lc substr $key, 0, 1;
138  } else {
139
73
49
    $key = lc $key;
140
73
45
    $key =~ s/''+/'/g;
141
73
38
    $key =~ s/'[sd]$//;
142
73
42
    $key =~ s/^[^Ii]?'+(.*)/$1/;
143
73
29
    $key =~ s/(.*?)'$/$1/;
144
73
63
    $char = substr $key, 0, 1;
145  }
146
89
110
  return ($key, $char);
147}
148
149sub load_expect {
150
13
13
476
  my ($expect) = @_;
151
13
6
  our %expected;
152
13
15
  %expected = ();
153
13
50
101
  if (open(EXPECT, '<:utf8', $expect)) {
154
13
59
    while (my $word = <EXPECT>) {
155
46
62
      $word =~ s/\R//;
156
46
84
      $expected{$word} = 0;
157    }
158
13
33
    close EXPECT;
159  }
160}
161
162sub harmonize_expect {
163
12
12
5
  our $disable_word_collating;
164
12
5
  our %letter_map;
165
12
6
  our %expected;
166
167
12
14
  for my $word (keys %expected) {
168
43
31
    my ($key, $char) = collate_key $word;
169
43
22
    my %word_map = ();
170
43
100
48
    next unless defined $letter_map{$char}{$key};
171
15
15
9
19
    %word_map = %{$letter_map{$char}{$key}};
172
15
100
17
    next if defined $word_map{$word};
173
3
3
    my $words = scalar keys %word_map;
174
3
50
1
    next if $words > 2;
175
3
100
3
    if ($word eq $key) {
176
1
50
1
      next if ($words > 1);
177    }
178
2
2
    delete $expected{$word};
179  }
180}
181
182sub group_related_words {
183
13
13
8
  our %letter_map;
184
13
4
  our $disable_word_collating;
185
13
100
10
  return if $disable_word_collating;
186
187  # group related words
188
11
25
  for my $char (sort CheckSpelling::Util::number_biased keys %letter_map) {
189
19
19
13
15
    for my $plural_key (sort keys(%{$letter_map{$char}})) {
190
22
13
      my $key = stem_word $plural_key;
191
22
100
16
      next if $key eq $plural_key;
192
4
100
7
      next unless defined $letter_map{$char}{$key};
193
3
3
1
4
      my %word_map = %{$letter_map{$char}{$key}};
194
3
3
2
3
      for my $word (keys(%{$letter_map{$char}{$plural_key}})) {
195
3
4
        $word_map{$word} = 1;
196      }
197
3
4
      $letter_map{$char}{$key} = \%word_map;
198
3
3
      delete $letter_map{$char}{$plural_key};
199    }
200  }
201}
202
203sub count_warning {
204
15
15
13
  my ($warning) = @_;
205
15
7
  our %counters;
206
15
7
  our %ignored_event_map;
207
15
100
22
  if ($warning =~ /\(([-\w]+)\)$/) {
208
9
7
    my ($code) = ($1);
209
9
50
8
    next if defined $ignored_event_map{$code};
210
9
10
    ++$counters{$code};
211  }
212}
213
214sub report_timing {
215
0
0
0
  my ($name, $start_time, $directory, $marker) = @_;
216
0
0
  my $end_time = (stat "$directory/$marker")[9];
217
0
0
  $name =~ s/"/\\"/g;
218
0
0
  print TIMING_REPORT "\"$name\", $start_time, $end_time\n";
219}
220
221sub get_pattern_with_context {
222
26
26
11
  my ($path) = @_;
223
26
100
32
  return unless defined $ENV{$path};
224
13
14
  $ENV{$path} =~ /(.*)/;
225
13
50
79
  return unless open ITEMS, '<:utf8', $1;
226
227
13
5
  my @items;
228
13
8
  my $context = '';
229
13
61
  while (<ITEMS>) {
230
2
2
    my $pattern = $_;
231
2
100
3
    if ($pattern =~ /^#/) {
232
1
50
2
      if ($pattern =~ /^# /) {
233
1
2
        $context .= $pattern;
234      } else {
235
0
0
        $context = '';
236      }
237
1
2
      next;
238    }
239
1
1
    chomp $pattern;
240
1
50
2
    unless ($pattern =~ /./) {
241
0
0
      $context = '';
242
0
0
      next;
243    }
244
1
1
    push @items, $context.$pattern;
245
1
4
    $context = '';
246  }
247
13
33
  close ITEMS;
248
13
14
  return @items;
249}
250
251sub summarize_totals {
252
26
26
23
  my ($formatter, $path, $items, $totals, $file_counts) = @_;
253
26
26
100
12
26
  return unless @{$totals};
254
1
50
20
  return unless open my $fh, '>:utf8', $path;
255
1
1
1
0
  my $totals_count = scalar(@{$totals}) - 1;
256
1
1
  my @indices;
257
1
50
1
  if ($file_counts) {
258    @indices = sort {
259
0
0
0
0
0
      $totals->[$b] <=> $totals->[$a] ||
260      $file_counts->[$b] <=> $file_counts->[$a]
261    } 0 .. $totals_count;
262  } else {
263    @indices = sort {
264
1
0
1
0
      $totals->[$b] <=> $totals->[$a]
265    } 0 .. $totals_count;
266  }
267
1
2
  for my $i (@indices) {
268
1
50
0
    last unless $totals->[$i] > 0;
269
1
1
    my $rule_with_context = $items->[$i];
270
1
1
    my ($description, $rule);
271
1
50
3
    if ($rule_with_context =~ /^(.*\n)([^\n]+)$/s) {
272
1
2
      ($description, $rule) = ($1, $2);
273    } else {
274
0
0
      ($description, $rule) = ('', $rule_with_context);
275    }
276
1
50
2
    print $fh $formatter->(
277      $totals->[$i],
278      ($file_counts ? " file-count: $file_counts->[$i]" : ""),
279      $description,
280      $rule
281    );
282  }
283
1
31
  close $fh;
284}
285
286sub get_special {
287
21
21
16
  my ($file, $special) = @_;
288
21
100
24
  return 'file-list' if $file eq $special->{'file_list'};
289
19
100
24
  return 'pr-title' if $file eq $special->{'pr_title_file'};
290
17
100
20
  return 'pr-description' if $file eq $special->{'pr_description_file'};
291
15
100
46
  return 'commit-message' if !rindex($file, $special->{'commit_messages'});
292
13
16
  return 'file';
293}
294
295sub main {
296
13
13
21549
  my @directories;
297  my @cleanup_directories;
298
13
0
  my @check_file_paths;
299
300
13
13
  my $early_warnings = CheckSpelling::Util::get_file_from_env('early_warnings', '/dev/null');
301
13
13
  my $warning_output = CheckSpelling::Util::get_file_from_env('warning_output', '/dev/stderr');
302
13
6
  my $more_warnings = CheckSpelling::Util::get_file_from_env('more_warnings', '/dev/stderr');
303
13
6
  my $counter_summary = CheckSpelling::Util::get_file_from_env('counter_summary', '/dev/stderr');
304
13
4
  my $ignored_events = CheckSpelling::Util::get_file_from_env('ignored_events', '');
305
13
100
9
  if ($ignored_events) {
306
6
2
    our %ignored_event_map;
307
6
8
    for my $event (split /,/, $ignored_events) {
308
6
6
      $ignored_event_map{$event} = 1;
309    }
310  }
311
13
9
  my $should_exclude_file = CheckSpelling::Util::get_file_from_env('should_exclude_file', '/dev/null');
312
13
11
  my $unknown_word_limit = CheckSpelling::Util::get_val_from_env('unknown_word_limit', undef);
313
13
12
  my $unknown_file_word_limit = CheckSpelling::Util::get_val_from_env('unknown_file_word_limit', undef);
314
13
10
  my $candidate_example_limit = CheckSpelling::Util::get_file_from_env('INPUT_CANDIDATE_EXAMPLE_LIMIT', '3');
315
13
12
  my $disable_flags = CheckSpelling::Util::get_file_from_env('INPUT_DISABLE_CHECKS', '');
316
13
10
  my $only_check_changed_files = CheckSpelling::Util::get_file_from_env('INPUT_ONLY_CHECK_CHANGED_FILES', '');
317
13
11
  my $disable_noisy_file = $disable_flags =~ /(?:^|,|\s)noisy-file(?:,|\s|$)/;
318
13
67
31
  our $disable_word_collating = $only_check_changed_files || $disable_flags =~ /(?:^|,|\s)word-collating(?:,|\s|$)/;
319
13
12
  my $file_list = CheckSpelling::Util::get_file_from_env('check_file_names', '');
320
13
12
  my $pr_title_file = CheckSpelling::Util::get_file_from_env('pr_title_file', '');
321
13
12
  my $pr_description_file = CheckSpelling::Util::get_file_from_env('pr_description_file', '');
322
13
11
  my $commit_messages = CheckSpelling::Util::get_file_from_env('commit_messages', '');
323
13
11
  my $timing_report = CheckSpelling::Util::get_file_from_env('timing_report', '');
324
13
18
  my $special = {
325    'file_list' => $file_list,
326    'pr_title_file' => $pr_title_file,
327    'pr_description_file' => $pr_description_file,
328    'commit_messages' => $commit_messages,
329  };
330
13
9
  my ($start_time, $end_time);
331
332
13
279
  open WARNING_OUTPUT, '>:utf8', $warning_output;
333
13
188
  open MORE_WARNINGS, '>:utf8', $more_warnings;
334
13
159
  open COUNTER_SUMMARY, '>:utf8', $counter_summary;
335
13
93
  open SHOULD_EXCLUDE, '>:utf8', $should_exclude_file;
336
13
50
12
  if ($timing_report) {
337
0
0
    open TIMING_REPORT, '>:utf8', $timing_report;
338
0
0
    print TIMING_REPORT "file, start, finish\n";
339  }
340
341
13
13
  my @candidates = get_pattern_with_context('candidates_path');
342
13
8
  my @candidate_totals = (0) x scalar @candidates;
343
13
5
  my @candidate_file_counts = (0) x scalar @candidates;
344
345
13
10
  my @forbidden = get_pattern_with_context('forbidden_path');
346
13
12
  my @forbidden_totals = (0) x scalar @forbidden;
347
348
13
4
  my @delayed_warnings;
349
13
24
  our %letter_map = ();
350
351
13
7
  my %file_map = ();
352
353
13
30
  for my $directory (<>) {
354
16
15
    chomp $directory;
355
16
50
25
    next unless $directory =~ /^(.*)$/;
356
16
14
    $directory = $1;
357
16
100
49
    unless (-e $directory) {
358
1
2
      print STDERR "Could not find: $directory\n";
359
1
2
      next;
360    }
361
15
100
35
    unless (-d $directory) {
362
1
11
      print STDERR "Not a directory: $directory\n";
363
1
1
      next;
364    }
365
366    # if there's no filename, we can't report
367
14
100
90
    next unless open(NAME, '<:utf8', "$directory/name");
368
13
59
    my $file=<NAME>;
369
13
25
    close NAME;
370
371
13
28
    $file_map{$file} = $directory;
372  }
373
374
13
22
  for my $file (sort keys %file_map) {
375
13
13
    my $directory = $file_map{$file};
376
13
50
11
    if ($timing_report) {
377
0
0
      $start_time = (stat "$directory/name")[9];
378    }
379
380
13
100
60
    if (-e "$directory/skipped") {
381
1
7
      open SKIPPED, '<:utf8', "$directory/skipped";
382
1
9
      my $reason=<SKIPPED>;
383
1
2
      close SKIPPED;
384
1
1
      chomp $reason;
385
1
3
      push @delayed_warnings, "$file:1:1 ... 1, Warning - Skipping `$file` because $reason\n";
386
1
13
      print SHOULD_EXCLUDE "$file\n";
387
1
1
      push @cleanup_directories, $directory;
388
1
50
1
      report_timing($file, $start_time, $directory, 'skipped') if ($timing_report);
389
1
2
      next;
390    }
391
392    # stats isn't written if there was nothing interesting in the file
393
12
100
34
    unless (-s "$directory/stats") {
394
1
50
1
      report_timing($file, $start_time, $directory, 'warnings') if ($timing_report);
395
1
1
      push @directories, $directory;
396
1
1
      next;
397    }
398
399
11
100
11
    if ($file eq $file_list) {
400
1
6
      open FILE_LIST, '<:utf8', $file_list;
401
1
1
      push @check_file_paths, '0 placeholder';
402
1
5
      for my $check_file_path (<FILE_LIST>) {
403
4
5
        chomp $check_file_path;
404
4
2
        push @check_file_paths, $check_file_path;
405      }
406
1
2
      close FILE_LIST;
407    }
408
409
11
8
    my ($words, $unrecognized, $unknown, $unique);
410
411    {
412
11
11
4
57
      open STATS, '<:utf8', "$directory/stats";
413
11
43
      my $stats=<STATS>;
414
11
21
      close STATS;
415
11
6
      $words=get_field($stats, 'words');
416
11
7
      $unrecognized=get_field($stats, 'unrecognized');
417
11
7
      $unknown=get_field($stats, 'unknown');
418
11
8
      $unique=get_field($stats, 'unique');
419
11
5
      my @candidate_list;
420
11
50
8
      if (@candidate_totals) {
421
0
0
        @candidate_list=get_array($stats, 'candidates');
422
0
0
        my @lines=get_array($stats, 'candidate_lines');
423
0
0
0
        if (@candidate_list) {
424
0
0
          for (my $i=0; $i < scalar @candidate_list; $i++) {
425
0
0
            my $hits = $candidate_list[$i];
426
0
0
0
            if ($hits) {
427
0
0
              $candidate_totals[$i] += $hits;
428
0
0
0
              if ($candidate_file_counts[$i]++ < $candidate_example_limit) {
429
0
0
                my $pattern = (split /\n/,$candidates[$i])[-1];
430
0
0
                my $position = $lines[$i];
431
0
0
                $position =~ s/:(\d+)$/ ... $1/;
432
0
0
                my $wrapped = CheckSpelling::Util::wrap_in_backticks($pattern);
433
0
0
                push @delayed_warnings, "$file:$position, Notice - Line matches candidate pattern $wrapped (candidate-pattern)\n";
434              }
435            }
436          }
437        }
438      }
439
11
100
11
      if (@forbidden_totals) {
440
1
1
        my @forbidden_list=get_array($stats, 'forbidden');
441
1
1
        my @lines=get_array($stats, 'forbidden_lines');
442
1
50
1
        if (@forbidden_list) {
443
1
1
          for (my $i=0; $i < scalar @forbidden_list; $i++) {
444
1
0
            my $hits = $forbidden_list[$i];
445
1
50
1
            if ($hits) {
446
1
3
              $forbidden_totals[$i] += $hits;
447            }
448          }
449        }
450      }
451      #print STDERR "$file (unrecognized: $unrecognized; unique: $unique; unknown: $unknown, words: $words, candidates: [".join(", ", @candidate_list)."])\n";
452    }
453
454
11
50
9
    report_timing($file, $start_time, $directory, 'unknown') if ($timing_report);
455
11
7
    my $kind = get_special($file, $special);
456    # These heuristics are very new and need tuning/feedback
457
11
100
14
    if (
458        ($unknown > $unique)
459        # || ($unrecognized > $words / 2)
460    ) {
461
1
50
1
      unless ($disable_noisy_file) {
462
1
50
1
        if ($kind eq 'file') {
463
1
2
          print SHOULD_EXCLUDE "$file\n";
464        }
465
1
1
        my $warning = "noisy-$kind";
466
1
1
        count_warning $warning;
467
1
2
        push @delayed_warnings, "$file:1:1 ... 1, Warning - Skipping `$file` because it seems to have more noise ($unknown) than unique words ($unique) (total: $unrecognized / $words). ($warning)\n";
468
1
0
        push @cleanup_directories, $directory;
469
1
2
        next;
470      }
471    }
472
10
7
    push @directories, $directory;
473
10
100
100
47
    unless ($kind =~ /^file/ && -s "$directory/unknown") {
474
4
7
      next;
475    }
476
6
37
    open UNKNOWN, '<:utf8', "$directory/unknown";
477
6
43
    for my $token (<UNKNOWN>) {
478
49
50
      $token =~ s/\R//;
479
49
100
63
      next unless $token =~ /./;
480
46
34
      my ($key, $char) = collate_key $token;
481
46
100
56
      $letter_map{$char} = () unless defined $letter_map{$char};
482
46
19
      my %word_map = ();
483
46
14
100
44
16
      %word_map = %{$letter_map{$char}{$key}} if defined $letter_map{$char}{$key};
484
46
39
      $word_map{$token} = 1;
485
46
54
      $letter_map{$char}{$key} = \%word_map;
486    }
487
6
23
    close UNKNOWN;
488  }
489
13
36
  close SHOULD_EXCLUDE;
490
13
50
11
  close TIMING_REPORT if $timing_report;
491
492  summarize_totals(
493    sub {
494
0
0
0
      my ($hits, $files, $context, $pattern) = @_;
495
0
0
      return "# hit-count: $hits$files\n$context$pattern\n\n",
496    },
497
13
32
    CheckSpelling::Util::get_file_from_env('candidate_summary', '/dev/stderr'),
498    \@candidates,
499    \@candidate_totals,
500    \@candidate_file_counts,
501  );
502
503  summarize_totals(
504    sub {
505
1
1
1
      my (undef, undef, $context, $pattern) = @_;
506
1
2
      $context =~ s/^# //gm;
507
1
1
      chomp $context;
508
1
1
      my $details;
509
1
50
3
      if ($context =~ /^(.*?)$(.*)/ms) {
510
1
1
        ($context, $details) = ($1, $2);
511
1
50
1
        $details = "\n$details" if $details;
512      }
513
1
50
1
      $context = 'Pattern' unless $context;
514
1
7
      return "#### $context$details\n```\n$pattern\n```\n\n";
515    },
516
13
34
    CheckSpelling::Util::get_file_from_env('forbidden_summary', '/dev/stderr'),
517    \@forbidden,
518    \@forbidden_totals,
519  );
520
521
13
33
  group_related_words;
522
523
13
100
14
  if (defined $ENV{'expect'}) {
524
12
10
    $ENV{'expect'} =~ /(.*)/;
525
12
12
    load_expect $1;
526
12
12
    harmonize_expect;
527  }
528
529
13
5
  my %seen = ();
530
13
8
  our %counters;
531
13
5
  %counters = ();
532
533
13
100
45
  if (-s $early_warnings) {
534
1
6
    open WARNINGS, '<:utf8', $early_warnings;
535
1
8
    for my $warning (<WARNINGS>) {
536
1
1
      chomp $warning;
537
1
1
      count_warning $warning;
538
1
50
1
      next if should_skip_warning $warning;
539
1
4
      print WARNING_OUTPUT "$warning\n";
540    }
541
1
3
    close WARNINGS;
542  }
543
544
13
6
  our %last_seen;
545
13
5
  my %unknown_file_word_count;
546
13
9
  for my $directory (@directories) {
547
11
100
34
    next unless (-s "$directory/warnings");
548
10
50
59
    next unless open(NAME, '<:utf8', "$directory/name");
549
10
34
    my $file=<NAME>;
550
10
20
    close NAME;
551
10
5
    my $kind = get_special($file, $special);
552
10
56
    open WARNINGS, '<:utf8', "$directory/warnings";
553
10
100
5
    if ($kind ne 'file-list') {
554
9
62
      for my $warning (<WARNINGS>) {
555
55
38
        chomp $warning;
556
55
100
102
        if ($warning =~ m/:(\d+):(\d+ \.\.\. \d+): `(.*)`/) {
557
51
49
          my ($line, $range, $item) = ($1, $2, $3);
558
51
36
          my $wrapped = CheckSpelling::Util::wrap_in_backticks($item);
559
51
20
          my $reason = 'unrecognized-spelling';
560
51
100
35
          $reason .= "-$kind" unless $kind eq 'file';
561
51
112
          $warning =~ s/:\d+:\d+ \.\.\. \d+: `.*`/:$line:$range, Warning - $wrapped is not a recognized word ($reason)/;
562
51
100
33
          next if log_skip_item($item, $file, $warning, $unknown_word_limit);
563
13
50
11
          count_warning $warning if $kind ne 'file';
564        } else {
565
4
50
3
          if ($warning =~ /\`(.*?)\` in line \(token-is-substring\)/) {
566
0
0
0
            next if skip_item($1);
567          }
568
4
3
          count_warning $warning;
569        }
570
17
50
13
        next if should_skip_warning $warning;
571
17
66
        print WARNING_OUTPUT "$file$warning\n";
572      }
573    } else {
574
1
7
      for my $warning (<WARNINGS>) {
575
6
7
        chomp $warning;
576
6
50
8
        next unless $warning =~ s/^:(\d+)/:1/;
577
6
5
        $file = $check_file_paths[$1];
578
6
100
12
        if ($warning =~ m/:(\d+ \.\.\. \d+): `(.*)`/) {
579
4
4
          my ($range, $item) = ($1, $2);
580
4
3
          my $wrapped = CheckSpelling::Util::wrap_in_backticks($item);
581
4
9
          $warning =~ s/:\d+ \.\.\. \d+: `.*`/:$range, Warning - $wrapped is not a recognized word (check-file-path)/;
582
4
50
4
          next if skip_item($item);
583
4
50
3
          if (defined $unknown_file_word_limit) {
584
4
100
6
            next if ++$unknown_file_word_count{$item} > $unknown_file_word_limit;
585          }
586        }
587
5
100
2
        next if should_skip_warning $warning;
588
4
13
        print WARNING_OUTPUT "$file$warning\n";
589
4
4
        count_warning $warning;
590      }
591    }
592
10
39
    close WARNINGS;
593  }
594
13
238
  close MORE_WARNINGS;
595
596
13
12
  for my $warning (@delayed_warnings) {
597
2
50
1
    next if should_skip_warning $warning;
598
2
2
    count_warning $warning;
599
2
3
    print WARNING_OUTPUT $warning;
600  }
601
13
100
9
  if (defined $unknown_word_limit) {
602
1
2
    for my $warned_word (sort keys %last_seen) {
603
1
50
2
      my $warning_count = $seen{$warned_word} || 0;
604
1
50
2
      next unless $warning_count >= $unknown_word_limit;
605
0
0
      my $warning = $last_seen{$warned_word};
606
0
0
      $warning =~ s/\Q (unrecognized-spelling)\E/ -- found $warning_count times (limited-references)\n/;
607
0
0
0
      next if should_skip_warning $warning;
608
0
0
      print WARNING_OUTPUT $warning;
609
0
0
      count_warning $warning;
610    }
611  }
612
13
260
  close WARNING_OUTPUT;
613
614
13
100
13
  if (%counters) {
615
3
2
    my $continue='';
616
3
4
    print COUNTER_SUMMARY "{\n";
617
3
6
    for my $code (sort keys %counters) {
618
5
7
      print COUNTER_SUMMARY qq<$continue"$code": $counters{$code}\n>;
619
5
3
      $continue=',';
620    }
621
3
3
    print COUNTER_SUMMARY "}\n";
622  }
623
13
82
  close COUNTER_SUMMARY;
624
625  # display the current unknown
626
13
30
  for my $char (sort keys %letter_map) {
627
46
46
21
122
    for my $key (sort CheckSpelling::Util::case_biased keys(%{$letter_map{$char}})) {
628
29
29
12
40
      my %word_map = %{$letter_map{$char}{$key}};
629
29
24
      my @words = keys(%word_map);
630
29
100
16
      if (scalar(@words) > 1) {
631
13
20
50
14
67
        print $key." (".(join ", ", sort { length($a) <=> length($b) || $a cmp $b } @words).")";
632      } else {
633
16
55
        print $words[0];
634      }
635
29
102
      print "\n";
636    }
637  }
638}
639
6401;