Natools

Check-in [1dce15910c]
Login
Overview
Comment:tools/smaz: genericize To_Dictionary
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA1: 1dce15910cd743af87a7a2a0e42bb1cda514ed2c
User & Date: nat on 2016-12-05 21:01:44
Other Links: manifest | tags
Context
2016-12-06
22:40
tools/smaz: refactor dictionary holder inside the generic package check-in: 1eac1c8b76 user: nat tags: trunk
2016-12-05
21:01
tools/smaz: genericize To_Dictionary check-in: 1dce15910c user: nat tags: trunk
2016-12-04
20:06
tools/smaz: genericize Optimize_Dictionary check-in: 5c617d9676 user: nat tags: trunk
Changes

Modified tools/smaz.adb from [20a034958d] to [d404854fda].

150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172

173
174
175
176
177
178
179












180
181
182
183
184
185
186
150
151
152
153
154
155
156






157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193







-
-
-
-
-
-










+







+
+
+
+
+
+
+
+
+
+
+
+








   procedure Process
     (Handler : in Callback'Class;
      Word_List : in Natools.Smaz_Tools.String_Lists.List;
      Data_List : in Natools.Smaz_Tools.String_Lists.List);
      --  Perform the requested operations

   function To_Dictionary
     (Handler : in Callback'Class;
      Input : in Natools.Smaz_Tools.String_Lists.List)
     return Natools.Smaz_256.Dictionary;
      --  Convert the input into a dictionary given the option in Handler

   procedure Use_Dictionary (Dict : in out Natools.Smaz_256.Dictionary);
      --  Update Dictionary.Hash so that it can be actually used



   generic
      type Dictionary (<>) is private;
      type Dictionary_Entry is (<>);
      type Methods is (<>);
      type String_Count is range <>;
      type Word_Counter is private;

      type Dictionary_Counts is array (Dictionary_Entry) of String_Count;

      with package Holders
        is new Ada.Containers.Indefinite_Holders (Dictionary);
      with package String_Lists
        is new Ada.Containers.Indefinite_Doubly_Linked_Lists (String);

      with procedure Add_Substrings
        (Counter : in out Word_Counter;
         Phrase : in String;
         Min_Size : in Positive;
         Max_Size : in Positive);

      with procedure Add_Words
        (Counter : in out Word_Counter;
         Phrase : in String;
         Min_Size : in Positive;
         Max_Size : in Positive);

      with function Append_String
        (Dict : in Dictionary;
         Element : in String)
        return Dictionary;

      with function Dict_Entry
196
197
198
199
200
201
202




203
204
205
206
207
208



















209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239











240
241
242
243
244
245
246
247
248
249









250
251
252
253
254
255
256
257
258
259







260
261
262
263
264
265
266
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258











259
260
261
262
263
264
265
266
267
268
269
270









271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303







+
+
+
+






+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+




















-
-
-
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+

-
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+










+
+
+
+
+
+
+








      with procedure Evaluate_Dictionary_Partial
        (Dict : in Dictionary;
         Corpus_Entry : in String;
         Compressed_Size : in out Ada.Streams.Stream_Element_Count;
         Counts : in out Dictionary_Counts);

      with procedure Filter_By_Count
        (Counter : in out Word_Counter;
         Threshold_Count : in String_Count);

      with function Last_Code (Dict : in Dictionary) return Dictionary_Entry;

      with function Remove_Element
        (Dict : in Dictionary;
         Element : in Dictionary_Entry)
        return Dictionary;

      with function Simple_Dictionary
        (Counter : in Word_Counter;
         Word_Count : in Natural;
         Method : in Methods)
        return String_Lists.List;

      with procedure Simple_Dictionary_And_Pending
        (Counter : in Word_Counter;
         Word_Count : in Natural;
         Selected : out String_Lists.List;
         Pending : out String_Lists.List;
         Method : in Methods;
         Max_Pending_Count : in Ada.Containers.Count_Type);

      with function To_Dictionary
        (List : in String_Lists.List;
         Variable_Length_Verbatim : in Boolean)
        return Dictionary;

      with procedure Use_Dictionary (Dict : in out Dictionary) is <>;

      with function Worst_Element
        (Dict : in Dictionary;
         Counts : in Dictionary_Counts;
         Method : in Methods)
        return Dictionary_Entry;

   package Dictionary_Subprograms is

      procedure Evaluate_Dictionary
        (Job_Count : in Natural;
         Dict : in Dictionary;
         Corpus : in String_Lists.List;
         Compressed_Size : out Ada.Streams.Stream_Element_Count;
         Counts : out Dictionary_Counts);
         --  Dispatch to parallel or non-parallel version of
         --  Evaluate_Dictionary depending on Job_Count.

         procedure Optimization_Round
           (Dict : in out Holders.Holder;
            Score : in out Ada.Streams.Stream_Element_Count;
            Counts : in out Dictionary_Counts;
            Pending_Words : in out String_Lists.List;
            Input_Texts : in String_Lists.List;
            Job_Count : in Natural;
            Method : in Methods;
            Updated : out Boolean);
         --  Try to improve on Dict by replacing a single entry from it with
         --  one of the substring in Pending_Words.
      procedure Optimization_Round
        (Dict : in out Holders.Holder;
         Score : in out Ada.Streams.Stream_Element_Count;
         Counts : in out Dictionary_Counts;
         Pending_Words : in out String_Lists.List;
         Input_Texts : in String_Lists.List;
         Job_Count : in Natural;
         Method : in Methods;
         Updated : out Boolean);
      --  Try to improve on Dict by replacing a single entry from it with
      --  one of the substring in Pending_Words.

         function Optimize_Dictionary
           (Base : in Dictionary;
            Pending_Words : in String_Lists.List;
            Input_Texts : in String_Lists.List;
            Job_Count : in Natural;
            Method : in Methods)
           return Dictionary;
         --  Optimize the dictionary on Input_Texts, starting with Base and
         --  adding substrings from Pending_Words.
      function Optimize_Dictionary
        (Base : in Dictionary;
         Pending_Words : in String_Lists.List;
         Input_Texts : in String_Lists.List;
         Job_Count : in Natural;
         Method : in Methods)
        return Dictionary;
      --  Optimize the dictionary on Input_Texts, starting with Base and
      --  adding substrings from Pending_Words.

      procedure Parallel_Evaluate_Dictionary
        (Job_Count : in Positive;
         Dict : in Dictionary;
         Corpus : in String_Lists.List;
         Compressed_Size : out Ada.Streams.Stream_Element_Count;
         Counts : out Dictionary_Counts);
         --  Return the same results as Natools.Smaz.Tools.Evaluate_Dictionary,
         --  but hopefully more quickly, using Job_Count tasks.

      function To_Dictionary
        (Handler : in Callback'Class;
         Input : in String_Lists.List;
         Method : in Methods)
        return Dictionary;
         --  Convert the input into a dictionary given the option in Handler

   end Dictionary_Subprograms;



   package body Dictionary_Subprograms is

      procedure Evaluate_Dictionary
469
470
471
472
473
474
475































































476
477
478
479
480
481
482
483
484

485
486
487


488
489
490
491

492
493




494
495
496
497
498
499
500
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608







+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+









+



+
+




+


+
+
+
+







         Cursor : String_Lists.Cursor := String_Lists.First (Corpus);
      begin
         Compressed_Size := 0;
         Counts := (others => 0);
         Parallel_Run (Cursor, Job_Count);
      end Parallel_Evaluate_Dictionary;


      function To_Dictionary
        (Handler : in Callback'Class;
         Input : in String_Lists.List;
         Method : in Methods)
        return Dictionary
      is
         use type Natools.Smaz_Tools.String_Count;
         use type Dict_Sources.Enum;
      begin
         case Handler.Dict_Source is
            when Dict_Sources.S_Expression =>
               return To_Dictionary (Input, Handler.Vlen_Verbatim);

            when Dict_Sources.Text_List | Dict_Sources.Unoptimized_Text_List =>
               declare
                  Counter : Word_Counter;
               begin
                  for S of Input loop
                     Add_Substrings
                       (Counter, S,
                        Handler.Min_Sub_Size, Handler.Max_Sub_Size);

                     if Handler.Max_Word_Size > Handler.Max_Sub_Size then
                        Add_Words
                          (Counter, S,
                           Handler.Max_Sub_Size + 1, Handler.Max_Word_Size);
                     end if;
                  end loop;

                  if Handler.Filter_Threshold > 0 then
                     Filter_By_Count
                       (Counter, String_Count (Handler.Filter_Threshold));
                  end if;

                  if Handler.Dict_Source = Dict_Sources.Text_List then
                     declare
                        Selected, Pending : String_Lists.List;
                     begin
                        Simple_Dictionary_And_Pending
                          (Counter,
                           Handler.Dict_Size,
                           Selected,
                           Pending,
                           Method,
                           Handler.Max_Pending);

                        return Optimize_Dictionary
                          (To_Dictionary (Selected, Handler.Vlen_Verbatim),
                           Pending,
                           Input,
                           Handler.Job_Count,
                           Method);
                     end;
                  else
                     return To_Dictionary
                       (Simple_Dictionary (Counter, Handler.Dict_Size, Method),
                        Handler.Vlen_Verbatim);
                  end if;
               end;
         end case;
      end To_Dictionary;

   end Dictionary_Subprograms;



   package Dict_256 is new Dictionary_Subprograms
     (Dictionary => Natools.Smaz_256.Dictionary,
      Dictionary_Entry => Ada.Streams.Stream_Element,
      Methods => Natools.Smaz_Tools.Methods.Enum,
      String_Count => Natools.Smaz_Tools.String_Count,
      Word_Counter => Natools.Smaz_Tools.Word_Counter,
      Dictionary_Counts => Tools_256.Dictionary_Counts,
      Holders => Holders,
      String_Lists => Natools.Smaz_Tools.String_Lists,
      Add_Substrings => Natools.Smaz_Tools.Add_Substrings,
      Add_Words => Natools.Smaz_Tools.Add_Words,
      Append_String => Tools_256.Append_String,
      Dict_Entry => Natools.Smaz_256.Dict_Entry,
      Evaluate_Dictionary => Tools_256.Evaluate_Dictionary,
      Evaluate_Dictionary_Partial => Tools_256.Evaluate_Dictionary_Partial,
      Filter_By_Count => Natools.Smaz_Tools.Filter_By_Count,
      Last_Code => Last_Code,
      Remove_Element => Tools_256.Remove_Element,
      Simple_Dictionary => Natools.Smaz_Tools.Simple_Dictionary,
      Simple_Dictionary_And_Pending
        => Natools.Smaz_Tools.Simple_Dictionary_And_Pending,
      To_Dictionary => Tools_256.To_Dictionary,
      Worst_Element => Tools_256.Worst_Index);



   overriding procedure Option
     (Handler  : in out Callback;
      Id       : in Options.Id;
819
820
821
822
823
824
825
826

827
828
829
830
831
832
833
927
928
929
930
931
932
933

934
935
936
937
938
939
940
941







-
+








   procedure Process
     (Handler : in Callback'Class;
      Word_List : in Natools.Smaz_Tools.String_Lists.List;
      Data_List : in Natools.Smaz_Tools.String_Lists.List)
   is
      Dictionary : Natools.Smaz_256.Dictionary
        := To_Dictionary (Handler, Word_List);
        := Dict_256.To_Dictionary (Handler, Word_List, Handler.Score_Method);
      Sx_Output : Natools.S_Expressions.Printers.Canonical
        (Ada.Text_IO.Text_Streams.Stream (Ada.Text_IO.Current_Output));
      Ada_Dictionary : constant String
        := Ada.Strings.Unbounded.To_String (Handler.Ada_Dictionary);
      Hash_Package : constant String
        := Ada.Strings.Unbounded.To_String (Handler.Hash_Package);
   begin
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1196
1197
1198
1199
1200
1201
1202































































1203
1204
1205
1206
1207
1208
1209







-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-







                       Tools_256.Score_Gain'Access);
                  end;
               end if;
            end;
      end case;
   end Process;


   function To_Dictionary
     (Handler : in Callback'Class;
      Input : in Natools.Smaz_Tools.String_Lists.List)
     return Natools.Smaz_256.Dictionary
   is
      use type Natools.Smaz_Tools.String_Count;
      use type Dict_Sources.Enum;
   begin
      case Handler.Dict_Source is
         when Dict_Sources.S_Expression =>
            return Tools_256.To_Dictionary (Input, Handler.Vlen_Verbatim);

         when Dict_Sources.Text_List | Dict_Sources.Unoptimized_Text_List =>
            declare
               Counter : Natools.Smaz_Tools.Word_Counter;
            begin
               for S of Input loop
                  Natools.Smaz_Tools.Add_Substrings
                    (Counter, S, Handler.Min_Sub_Size, Handler.Max_Sub_Size);

                  if Handler.Max_Word_Size > Handler.Max_Sub_Size then
                     Natools.Smaz_Tools.Add_Words
                       (Counter, S,
                        Handler.Max_Sub_Size + 1, Handler.Max_Word_Size);
                  end if;
               end loop;

               if Handler.Filter_Threshold > 0 then
                  Natools.Smaz_Tools.Filter_By_Count
                    (Counter, Handler.Filter_Threshold);
               end if;

               if Handler.Dict_Source = Dict_Sources.Text_List then
                  declare
                     Selected, Pending : Natools.Smaz_Tools.String_Lists.List;
                  begin
                     Natools.Smaz_Tools.Simple_Dictionary_And_Pending
                       (Counter,
                        Handler.Dict_Size,
                        Selected,
                        Pending,
                        Handler.Score_Method,
                        Handler.Max_Pending);

                     return Dict_256.Optimize_Dictionary
                       (Tools_256.To_Dictionary
                          (Selected, Handler.Vlen_Verbatim),
                        Pending,
                        Input,
                        Handler.Job_Count,
                        Handler.Score_Method);
                  end;
               else
                  return Tools_256.To_Dictionary
                    (Natools.Smaz_Tools.Simple_Dictionary
                       (Counter, Handler.Dict_Size, Handler.Score_Method),
                     Handler.Vlen_Verbatim);
               end if;
            end;
      end case;
   end To_Dictionary;


   procedure Use_Dictionary (Dict : in out Natools.Smaz_256.Dictionary) is
   begin
      Natools.Smaz_Tools.Set_Dictionary_For_Trie_Search
        (Tools_256.To_String_List (Dict));
      Dict.Hash := Natools.Smaz_Tools.Trie_Search'Access;