Many hyperlinks are disabled.
Use anonymous login
to enable hyperlinks.
Overview
Comment: | tools/smaz: add selection of the scoring method in optimization |
---|---|
Timelines: | family | ancestors | descendants | both | trunk |
Files: | files | file ages | folders |
SHA1: |
6877c806ab8df40620ed6f27a8fd3315 |
User & Date: | nat 2016-10-31 22:53:58 |
Context
2016-11-01
| ||
21:32 | tools/smaz: fix option arguments in help text check-in: 8a29fd3a72 user: nat tags: trunk | |
2016-10-31
| ||
22:53 | tools/smaz: add selection of the scoring method in optimization check-in: 6877c806ab user: nat tags: trunk | |
2016-10-30
| ||
18:19 | tools/sxcat: add a command-line option to output a list of input atoms check-in: ac88f5abfb user: nat tags: trunk | |
Changes
Changes to tools/smaz.adb.
︙ | ︙ | |||
52 53 54 55 56 57 58 59 60 61 62 63 64 65 | package Dict_Sources is type Enum is (S_Expression, Text_List, Unoptimized_Text_List); end Dict_Sources; package Options is type Id is (Output_Ada_Dict, Dictionary_Input, Decode, Encode, | > > > > | 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 | package Dict_Sources is type Enum is (S_Expression, Text_List, Unoptimized_Text_List); end Dict_Sources; package Methods is type Enum is (Encoded, Frequency, Gain); end Methods; package Options is type Id is (Output_Ada_Dict, Dictionary_Input, Decode, Encode, |
︙ | ︙ | |||
73 74 75 76 77 78 79 | Max_Sub_Size, Stat_Output, No_Stat_Output, Text_List_Input, Fast_Text_Input, Max_Word_Size, Sx_Output, | | > > | 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 | Max_Sub_Size, Stat_Output, No_Stat_Output, Text_List_Input, Fast_Text_Input, Max_Word_Size, Sx_Output, No_Sx_Output, Score_Method); end Options; package Getopt is new Natools.Getopt_Long (Options.Id); type Callback is new Getopt.Handlers.Callback with record Display_Help : Boolean := False; Need_Dictionary : Boolean := False; Stat_Output : Boolean := False; Sx_Output : Boolean := False; Sx_Dict_Output : Boolean := False; Min_Sub_Size : Positive := 1; Max_Sub_Size : Positive := 3; Max_Word_Size : Positive := 10; Job_Count : Natural := 0; Filter_Threshold : Natools.Smaz.Tools.String_Count := 0; Score_Method : Methods.Enum := Methods.Encoded; Action : Actions.Enum := Actions.Nothing; Ada_Dictionary : Ada.Strings.Unbounded.Unbounded_String; Hash_Package : Ada.Strings.Unbounded.Unbounded_String; Dict_Source : Dict_Sources.Enum := Dict_Sources.S_Expression; end record; overriding procedure Option |
︙ | ︙ | |||
118 119 120 121 122 123 124 | Counts : out Natools.Smaz.Tools.Dictionary_Counts); -- Dispatch to parallel or non-parallel version of Evaluate_Dictionary -- depending on Job_Count. function Getopt_Config return Getopt.Configuration; -- Build the configuration object | < < < < < < < < < < < > > > > > > > > > > > > | > | 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 | Counts : out Natools.Smaz.Tools.Dictionary_Counts); -- Dispatch to parallel or non-parallel version of Evaluate_Dictionary -- depending on Job_Count. function Getopt_Config return Getopt.Configuration; -- Build the configuration object function Length (Dictionary : in Natools.Smaz.Dictionary; E : in Ada.Streams.Stream_Element) return Score_Value is (Natools.Smaz.Dict_Entry (Dictionary, E)'Length); -- Length of a dictionary entry procedure Optimization_Round (Dict : in out Holders.Holder; Score : in out Ada.Streams.Stream_Element_Count; Counts : in out Natools.Smaz.Tools.Dictionary_Counts; Pending_Words : in out Natools.Smaz.Tools.String_Lists.List; Input_Texts : in Natools.Smaz.Tools.String_Lists.List; Job_Count : in Natural; Method : in Methods.Enum; Updated : out Boolean); -- Try to improve on Dict by replacing a single entry from it with -- one of the substring in Pending_Words. function Optimize_Dictionary (Base : in Natools.Smaz.Dictionary; Pending_Words : in Natools.Smaz.Tools.String_Lists.List; Input_Texts : in Natools.Smaz.Tools.String_Lists.List; Job_Count : in Natural; Method : in Methods.Enum) return Natools.Smaz.Dictionary; -- Optimize the dictionary on Input_Texts, starting with Base and -- adding substrings from Pending_Words. procedure Parallel_Evaluate_Dictionary (Job_Count : in Positive; Dict : in Natools.Smaz.Dictionary; |
︙ | ︙ | |||
193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 | (Dictionary : in Natools.Smaz.Dictionary; Counts : in Natools.Smaz.Tools.Dictionary_Counts; E : Ada.Streams.Stream_Element) return Score_Value is (Score_Value (Counts (E)) * (Length (Dictionary, E) - 1)); -- Score value using the number of bytes saved using E function To_Dictionary (Handler : in Callback'Class; Input : in Natools.Smaz.Tools.String_Lists.List) return Natools.Smaz.Dictionary; -- Convert the input into a dictionary given the option in Handler function Worst_Index (Dict : in Natools.Smaz.Dictionary; | > > > > > > > > > > > > | > | 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 | (Dictionary : in Natools.Smaz.Dictionary; Counts : in Natools.Smaz.Tools.Dictionary_Counts; E : Ada.Streams.Stream_Element) return Score_Value is (Score_Value (Counts (E)) * (Length (Dictionary, E) - 1)); -- Score value using the number of bytes saved using E function Score (Dictionary : in Natools.Smaz.Dictionary; Counts : in Natools.Smaz.Tools.Dictionary_Counts; E : in Ada.Streams.Stream_Element; Method : in Methods.Enum) return Score_Value is (case Method is when Methods.Encoded => Score_Encoded (Dictionary, Counts, E), when Methods.Frequency => Score_Frequency (Dictionary, Counts, E), when Methods.Gain => Score_Gain (Dictionary, Counts, E)); -- Scare value with dynamically chosen method function To_Dictionary (Handler : in Callback'Class; Input : in Natools.Smaz.Tools.String_Lists.List) return Natools.Smaz.Dictionary; -- Convert the input into a dictionary given the option in Handler function Worst_Index (Dict : in Natools.Smaz.Dictionary; Counts : in Natools.Smaz.Tools.Dictionary_Counts; Method : in Methods.Enum) return Ada.Streams.Stream_Element; -- Remove the worstly-scored item from Dict overriding procedure Option (Handler : in out Callback; Id : in Options.Id; |
︙ | ︙ | |||
283 284 285 286 287 288 289 290 291 292 293 294 295 296 | when Options.Job_Count => Handler.Job_Count := Natural'Value (Argument); when Options.Filter_Threshold => Handler.Filter_Threshold := Natools.Smaz.Tools.String_Count'Value (Argument); end case; end Option; procedure Evaluate_Dictionary (Job_Count : in Natural; Dict : in Natools.Smaz.Dictionary; | > > > | 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 | when Options.Job_Count => Handler.Job_Count := Natural'Value (Argument); when Options.Filter_Threshold => Handler.Filter_Threshold := Natools.Smaz.Tools.String_Count'Value (Argument); when Options.Score_Method => Handler.Score_Method := Methods.Enum'Value (Argument); end case; end Option; procedure Evaluate_Dictionary (Job_Count : in Natural; Dict : in Natools.Smaz.Dictionary; |
︙ | ︙ | |||
347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 | R.Add_Option ("stats", 's', No_Argument, Stat_Output); R.Add_Option ("no-stats", 'S', No_Argument, No_Stat_Output); R.Add_Option ("text-list", 't', No_Argument, Text_List_Input); R.Add_Option ("fast-text-list", 'T', No_Argument, Fast_Text_Input); R.Add_Option ("max-word-len", 'W', Required_Argument, Max_Word_Size); R.Add_Option ("s-expr", 'x', No_Argument, Sx_Output); R.Add_Option ("no-s-expr", 'X', No_Argument, No_Sx_Output); return R; end Getopt_Config; procedure Optimization_Round (Dict : in out Holders.Holder; Score : in out Ada.Streams.Stream_Element_Count; Counts : in out Natools.Smaz.Tools.Dictionary_Counts; Pending_Words : in out Natools.Smaz.Tools.String_Lists.List; Input_Texts : in Natools.Smaz.Tools.String_Lists.List; Job_Count : in Natural; Updated : out Boolean) is use type Ada.Streams.Stream_Element_Offset; New_Value : Ada.Strings.Unbounded.Unbounded_String; New_Position : Natools.Smaz.Tools.String_Lists.Cursor; Worst_Index : constant Ada.Streams.Stream_Element | > > | | 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 | R.Add_Option ("stats", 's', No_Argument, Stat_Output); R.Add_Option ("no-stats", 'S', No_Argument, No_Stat_Output); R.Add_Option ("text-list", 't', No_Argument, Text_List_Input); R.Add_Option ("fast-text-list", 'T', No_Argument, Fast_Text_Input); R.Add_Option ("max-word-len", 'W', Required_Argument, Max_Word_Size); R.Add_Option ("s-expr", 'x', No_Argument, Sx_Output); R.Add_Option ("no-s-expr", 'X', No_Argument, No_Sx_Output); R.Add_Option ("score-method", Required_Argument, Score_Method); return R; end Getopt_Config; procedure Optimization_Round (Dict : in out Holders.Holder; Score : in out Ada.Streams.Stream_Element_Count; Counts : in out Natools.Smaz.Tools.Dictionary_Counts; Pending_Words : in out Natools.Smaz.Tools.String_Lists.List; Input_Texts : in Natools.Smaz.Tools.String_Lists.List; Job_Count : in Natural; Method : in Methods.Enum; Updated : out Boolean) is use type Ada.Streams.Stream_Element_Offset; New_Value : Ada.Strings.Unbounded.Unbounded_String; New_Position : Natools.Smaz.Tools.String_Lists.Cursor; Worst_Index : constant Ada.Streams.Stream_Element := Smaz.Worst_Index (Dict.Element, Counts, Method); Worst_Value : constant String := Natools.Smaz.Dict_Entry (Dict.Element, Worst_Index); Worst_Count : constant Natools.Smaz.Tools.String_Count := Counts (Worst_Index); Base : constant Natools.Smaz.Dictionary := Natools.Smaz.Tools.Remove_Element (Dict.Element, Worst_Index); Old_Score : constant Ada.Streams.Stream_Element_Count := Score; |
︙ | ︙ | |||
426 427 428 429 430 431 432 | end Optimization_Round; function Optimize_Dictionary (Base : in Natools.Smaz.Dictionary; Pending_Words : in Natools.Smaz.Tools.String_Lists.List; Input_Texts : in Natools.Smaz.Tools.String_Lists.List; | | > > | 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 | end Optimization_Round; function Optimize_Dictionary (Base : in Natools.Smaz.Dictionary; Pending_Words : in Natools.Smaz.Tools.String_Lists.List; Input_Texts : in Natools.Smaz.Tools.String_Lists.List; Job_Count : in Natural; Method : in Methods.Enum) return Natools.Smaz.Dictionary is Holder : Holders.Holder := Holders.To_Holder (Base); Pending : Natools.Smaz.Tools.String_Lists.List := Pending_Words; Score : Ada.Streams.Stream_Element_Count; Counts : Natools.Smaz.Tools.Dictionary_Counts; Running : Boolean := True; begin Evaluate_Dictionary (Job_Count, Base, Input_Texts, Score, Counts); while Running loop Optimization_Round (Holder, Score, Counts, Pending, Input_Texts, Job_Count, Method, Running); end loop; return Holder.Element; end Optimize_Dictionary; |
︙ | ︙ | |||
702 703 704 705 706 707 708 709 710 711 712 713 714 715 | when Options.Filter_Threshold => Put_Line (Output, " <threshold>"); Put_Line (Output, Indent & Indent & "Before building a dictionary from substrings, remove"); Put_Line (Output, Indent & Indent & "substrings whose count is below the threshold."); end case; end loop; end Print_Help; function To_Dictionary (Handler : in Callback'Class; | > > > > > > | 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 | when Options.Filter_Threshold => Put_Line (Output, " <threshold>"); Put_Line (Output, Indent & Indent & "Before building a dictionary from substrings, remove"); Put_Line (Output, Indent & Indent & "substrings whose count is below the threshold."); when Options.Score_Method => Put_Line (Output, " <method>"); Put_Line (Output, Indent & Indent & "Select heuristic method to replace dictionary items" & " during optimization"); end case; end loop; end Print_Help; function To_Dictionary (Handler : in Callback'Class; |
︙ | ︙ | |||
750 751 752 753 754 755 756 | Natools.Smaz.Tools.Simple_Dictionary_And_Pending (Counter, 254, Selected, Pending); return Optimize_Dictionary (Natools.Smaz.Tools.To_Dictionary (Selected, True), Pending, Input, | | > | > | | 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 | Natools.Smaz.Tools.Simple_Dictionary_And_Pending (Counter, 254, Selected, Pending); return Optimize_Dictionary (Natools.Smaz.Tools.To_Dictionary (Selected, True), Pending, Input, Handler.Job_Count, Handler.Score_Method); end; else return Natools.Smaz.Tools.To_Dictionary (Natools.Smaz.Tools.Simple_Dictionary (Counter, 254), True); end if; end; end case; end To_Dictionary; function Worst_Index (Dict : in Natools.Smaz.Dictionary; Counts : in Natools.Smaz.Tools.Dictionary_Counts; Method : in Methods.Enum) return Ada.Streams.Stream_Element is Result : Ada.Streams.Stream_Element := 0; Worst_Score : Score_Value := Score_Encoded (Dict, Counts, 0); S : Score_Value; begin for I in 1 .. Dict.Dict_Last loop S := Score (Dict, Counts, I, Method); if S < Worst_Score then Result := I; Worst_Score := S; end if; end loop; |
︙ | ︙ |