redis4.0_ziplist源码剖析

1.ziplist

1.1.ziplist概述

The ziplist is a specially encoded dually linked list that is designed to be very memory efficient. It stores both strings and integer values, where integers are encoded as actual integers(integer内存表示) instead of a series of characters(ASCII码). It allows push and pop operations on either side of the list in O(1) time. However, because every operation requires a reallocation of the memory used by the ziplist, the actual complexity is related to the amount of memory used by the ziplist.

1.2.ziplist整体内存布局

1
2
3
4
5
6
7
|-----------|----------|---------|---------|---------|----|----------|---------|
| <zlbytes> | <zltail> | <zllen> | <entry0>| <entry1>| ...| <entryn> | <zlend> |
|-----------|----------|---------|---------|---------|----|----------|---------|
|低地址 <---------------------------- 高地址 |
NOTE: all fields are stored in little endian(即:低位在地地址,高位在高地址), if not specified otherwise.
  • The general layout of the ziplist is as follows:
    • 用于记录整个ziplist占用的内存字节数;
    • 用于记录最后一个节点(the last entry)的首地址距离ziplist首地址的偏移量;
    • 用于记录ziplist中节点(entry)数量;
    • 用于表示ziplist的结尾,为定制 0xFF;
1.2.ziplist中entry内存布局
1
2
3
4
5
6
7
|--------|----------|---------|
| prelen | encoding | content |
|--------|----------|---------|
|低地址 <---------- 高地址 |
NOTE:a complete entry is stored like above(content长度可能为0,即一个entry可能只有prelen和encoding字段,没有content字段).
1.2.1 entry每个字段的含义:
  • prevlen字段用于记录前一个ziplist节点(entry)的长度,该字段以字节为单位。
    • 如果前一个节点的长度小于255个字节,则prelen属性的长度为1bytes,该字节用于保存前一个节点的长度;
    • 如果前一个节点的长度大于或等于255个字节,则prelen字段的长度为5bytes,其中第一个bytes为定制0xFF,之后的四个bytes用于保存前一个节点的长度;
1.2.2 encoding字段的含义:
  • encoding字段用于表示content字段所保存的数据的类型(string or int)以及content的长度。
  • encoding表示string类型及长度:
    • 当encoding表示string类型时,encoding长度可能为1bytes,2bytes,5bytes,其中最低地址的头两个bits为非11(即(?? & 11) < 11)
    • 当encoding编码为 00bb bbbb(二进制) 时,encoding字段的长度为1bytes,content的长度小于等于0x3F的字节数组;
    • 当encoding编码为 01bb bbbb bbbb bbbb(二进制) 时,encoding字段的长度为2bytes(其中encoding中bb bbbb bbbb bbbb表示string的长度,即content的长度,采用big endian表示),content的长度为小于等于0x3FFF的字节数组;
    • 当encoding编码为10 __ aaaa aaaa bbbb bbbb cccc cccc dddd dddd (二进制)时,encoding字段的长度为5bytes(其中encoding中aaaaaaaa bbbbbbbb cccccccc dddddddd表示string的长度,采用big endian表示),content的长度为小于等于0xFFFFFFFF的字节数组;
  • encoding表示int*_t类型及长度:
    • 当encoding表示integer类型时,encoding字段的长度为1bytes,最低地址的头两个bits为11(即encoding为(11?? ????));
    • 当encoding的编码为 1100 0000(二进制)时,content的长度与2bytes,即表示int16_t类型
    • 当encoding的编码为 1101 0000(二进制)时,content的长度与4bytes,即表示int32_t类型
    • 当encoding的编码为 1110 0000(二进制)时,content的长度与8bytes,即表示int64_t类型
    • 当encoding的编码为 1110 0000(二进制)时,content的长度与3bytes,即表示int24_t类型
    • 当encoding的编码为 1111 1110(二进制)时,content的长度与1bytes,即表示int8_t类型
    • 当encoding的编码为 1100 xxxx(二进制)时,content的长度与0bytes,即表示immediate 4 bit integer,其中xxxx 4bits表示0,12;
    • |11111111| - End of ziplist special entry.

2.ziplist.c.源码分析

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
/* The ziplist is a specially encoded dually linked list that is designed
* to be very memory efficient. It stores both strings and integer values,
* where integers are encoded as actual integers instead of a series of
* characters. It allows push and pop operations on either side of the list
* in O(1) time. However, because every operation requires a reallocation of
* the memory used by the ziplist, the actual complexity is related to the
* amount of memory used by the ziplist.
*
* ----------------------------------------------------------------------------
*
* ZIPLIST OVERALL LAYOUT
* ======================
*
* The general layout of the ziplist is as follows:
*
* <zlbytes> <zltail> <zllen> <entry> <entry> ... <entry> <zlend>
*
* NOTE:如果没有特殊说明,所有fields采用小端法表示,即:低位在低地址。
* NOTE: all fields are stored in little endian, if not specified otherwise.
*
* <uint32_t zlbytes> is an unsigned integer to hold the number of bytes that
* the ziplist occupies, including the four bytes of the zlbytes field itself.
* This value needs to be stored to be able to resize the entire structure
* without the need to traverse it first.
*
* <uint32_t zltail> is the offset to the last entry in the list. This allows
* a pop operation on the far side of the list without the need for full
* traversal.
*
* <uint16_t zllen> is the number of entries. When there are more than
* 2^16-2 entires, this value is set to 2^16-1 and we need to traverse the
* entire list to know how many items it holds.
*
* <uint8_t zlend> is a special entry representing the end of the ziplist.
* Is encoded as a single byte equal to 255. No other normal entry starts
* with a byte set to the value of 255.
*
* ZIPLIST ENTRIES
* ===============
*
* Every entry in the ziplist is prefixed by metadata that contains two pieces
* of information. First, the length of the previous entry is stored to be
* able to traverse the list from back to front. Second, the entry encoding is
* provided. It represents the entry type, integer or string, and in the case
* of strings it also represents the length of the string payload.
* So a complete entry is stored like this:
*
* <prevlen> <encoding> <entry-data>
*
* Sometimes the encoding represents the entry itself, like for small integers
* as we'll see later. In such a case the <entry-data> part is missing, and we
* could have just:
*
* <prevlen> <encoding>
*
* The length of the previous entry, <prevlen>, is encoded in the following way:
* If this length is smaller than 255 bytes, it will only consume a single
* byte representing the length as an unsinged 8 bit integer. When the length
* is greater than or equal to 255, it will consume 5 bytes. The first byte is
* set to 255 (FF) to indicate a larger value is following. The remaining 4
* bytes take the length of the previous entry as value.
*
* So practically an entry is encoded in the following way:
*
* <prevlen from 0 to 254> <encoding> <entry>
*
* Or alternatively if the previous entry length is greater than 254 bytes
* the following encoding is used:
*
* 0xFF <4 bytes unsigned little endian prevlen> <encoding> <entry>
*
* The encoding field of the entry depends on the content of the
* entry. When the entry is a string, the first 2 bits of the encoding first
* byte will hold the type of encoding used to store the length of the string,
* followed by the actual length of the string. When the entry is an integer
* the first 2 bits are both set to 1. The following 2 bits are used to specify
* what kind of integer will be stored after this header. An overview of the
* different types and encodings is as follows. The first byte is always enough
* to determine the kind of entry.
*
* |00pppppp| - 1 byte
* String value with length less than or equal to 63 bytes (6 bits).
* "pppppp" represents the unsigned 6 bit length.
* |01pppppp|qqqqqqqq| - 2 bytes
* String value with length less than or equal to 16383 bytes (14 bits).
* IMPORTANT: The 14 bit number is stored in big endian.
* |10000000|qqqqqqqq|rrrrrrrr|ssssssss|tttttttt| - 5 bytes
* String value with length greater than or equal to 16384 bytes.
* Only the 4 bytes following the first byte represents the length
* up to 32^2-1. The 6 lower bits of the first byte are not used and
* are set to zero.
* IMPORTANT: The 32 bit number is stored in big endian.
* |11000000| - 3 bytes
* Integer encoded as int16_t (2 bytes).
* |11010000| - 5 bytes
* Integer encoded as int32_t (4 bytes).
* |11100000| - 9 bytes
* Integer encoded as int64_t (8 bytes).
* |11110000| - 4 bytes
* Integer encoded as 24 bit signed (3 bytes).
* |11111110| - 2 bytes
* Integer encoded as 8 bit signed (1 byte).
* |1111xxxx| - (with xxxx between 0000 and 1101) immediate 4 bit integer.
* Unsigned integer from 0 to 12. The encoded value is actually from
* 1 to 13 because 0000 and 1111 can not be used, so 1 should be
* subtracted from the encoded 4 bit value to obtain the right value.
* |11111111| - End of ziplist special entry.
*
* Like for the ziplist header, all the integers(int_*) are represented in little
* endian byte order, even when this code is compiled in big endian systems.
*
* EXAMPLES OF ACTUAL ZIPLISTS
* ===========================
*
* The following is a ziplist containing the two elements representing
* the strings "2" and "5". It is composed of 15 bytes, that we visually
* split into sections:
*
* [0f 00 00 00] [0c 00 00 00] [02 00] [00 f3] [02 f6] [ff]
* | | | | | |
* zlbytes zltail entries "2" "5" end
*
* The first 4 bytes represent the number 15, that is the number of bytes
* the whole ziplist is composed of. The second 4 bytes are the offset
* at which the last ziplist entry is found, that is 12, in fact the
* last entry, that is "5", is at offset 12 inside the ziplist.
* The next 16 bit integer represents the number of elements inside the
* ziplist, its value is 2 since there are just two elements inside.
* Finally "00 f3" is the first entry representing the number 2. It is
* composed of the previous entry length, which is zero because this is
* our first entry, and the byte F3 which corresponds to the encoding
* |1111xxxx| with xxxx between 0001 and 1101. We need to remove the "F"
* higher order bits 1111, and subtract 1 from the "3", so the entry value
* is "2". The next entry has a prevlen of 02, since the first entry is
* composed of exactly two bytes. The entry itself, F6, is encoded exactly
* like the first entry, and 6-1 = 5, so the value of the entry is 5.
* Finally the special entry FF signals the end of the ziplist.
*
* Adding another element to the above string with the value "Hello World"
* allows us to show how the ziplist encodes small strings. We'll just show
* the hex dump of the entry itself. Imagine the bytes as following the
* entry that stores "5" in the ziplist above:
*
* [02] [0b] [48 65 6c 6c 6f 20 57 6f 72 6c 64]
*
* The first byte, 02, is the length of the previous entry. The next
* byte represents the encoding in the pattern |00pppppp| that means
* that the entry is a string of length <pppppp>, so 0B means that
* an 11 bytes string follows. From the third byte (48) to the last (64)
* there are just the ASCII characters for "Hello World".
*
* ----------------------------------------------------------------------------
*
* Copyright (c) 2009-2012, Pieter Noordhuis <pcnoordhuis at gmail dot com>
* Copyright (c) 2009-2017, Salvatore Sanfilippo <antirez at gmail dot com>
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* * Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* * Neither the name of Redis nor the names of its contributors may be used
* to endorse or promote products derived from this software without
* specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <stdint.h>
#include <limits.h>
#include "zmalloc.h"
#include "util.h"
#include "ziplist.h"
#include "endianconv.h"
#include "redisassert.h"
//zlend的值
#define ZIP_END 255 /* Special "end of ziplist" entry. */
//该值用于判断prelen 字段的长度是1byte or 5 bytes
#define ZIP_BIG_PREVLEN 254 /* Max number of bytes of the previous entry, for
the "prevlen" field prefixing each entry, to be
represented with just a single byte. Otherwise
it is represented as FF AA BB CC DD, where
AA BB CC DD are a 4 bytes unsigned integer
representing the previous entry len. */
/* Different encoding/length possibilities */
//用于判断entry是str还是int
#define ZIP_STR_MASK 0xc0
//用于判断int类型(int_16, int_24, int_32, int_64)
#define ZIP_INT_MASK 0x30
#define ZIP_STR_06B (0 << 6) //00 000000
#define ZIP_STR_14B (1 << 6) //01 000000
#define ZIP_STR_32B (2 << 6) //10 000000
#define ZIP_INT_16B (0xc0 | 0<<4) //1100 0000
#define ZIP_INT_32B (0xc0 | 1<<4) //1101 0000
#define ZIP_INT_64B (0xc0 | 2<<4) //1110 0000
#define ZIP_INT_24B (0xc0 | 3<<4) //1111 0000
#define ZIP_INT_8B 0xfe //1111 1110
/* 4 bit integer immediate encoding |1111xxxx| with xxxx between
* 0001 and 1101. */
//用于获取immedate 4 bit integer的值
#define ZIP_INT_IMM_MASK 0x0f /* Mask to extract the 4 bits value. To add
one is needed to reconstruct the value. */
//最小值为(0xF1&0x0F) - 1
#define ZIP_INT_IMM_MIN 0xf1 /* 11110001 */
//最大值为(0xFD&0x0F) - 1
#define ZIP_INT_IMM_MAX 0xfd /* 11111101 */
//24bits表示有符号整数,24bits解析为有符号整数,如果感兴趣,请浏览http://qinch.github.io/2016/01/10/bits-int/
#define INT24_MAX 0x7fffff
#define INT24_MIN (-INT24_MAX - 1)
/* Macro to determine if the entry is a string. String entries never start
* with "11" as most significant bits of the first byte. */
//判断encoding编码是不是string类型
#define ZIP_IS_STR(enc) (((enc) & ZIP_STR_MASK) < ZIP_STR_MASK)
/* Utility macros.*/
/* Return total bytes a ziplist is composed of. */
//获得ziplist的总长度,单位为byte
#define ZIPLIST_BYTES(zl) (*((uint32_t*)(zl)))
/* Return the offset of the last item inside the ziplist. */
//最后一个entry首地址距离ziplist首地址的偏移量
#define ZIPLIST_TAIL_OFFSET(zl) (*((uint32_t*)((zl)+sizeof(uint32_t))))
/* Return the length of a ziplist, or UINT16_MAX if the length cannot be
* determined without scanning the whole ziplist. */
#define ZIPLIST_LENGTH(zl) (*((uint16_t*)((zl)+sizeof(uint32_t)*2)))
/* The size of a ziplist header: two 32 bit integers for the total
* bytes count and last item offset. One 16 bit integer for the number
* of items field. */
//ziplist header(zlbytes+zltail+zllen)大小,单位byte
#define ZIPLIST_HEADER_SIZE (sizeof(uint32_t)*2+sizeof(uint16_t))
/* Size of the "end of ziplist" entry. Just one byte. */
//zlend大小,单位byte
#define ZIPLIST_END_SIZE (sizeof(uint8_t))
/* Return the pointer to the first entry of a ziplist. */
//第一个entry的首地址
#define ZIPLIST_ENTRY_HEAD(zl) ((zl)+ZIPLIST_HEADER_SIZE)
/* Return the pointer to the last entry of a ziplist, using the
* last entry offset inside the ziplist header. */
//intrev32ifbe的作用:
//因为zltail字段采用littele endian表示,所以
//1:如果为BYTE_ORDER(在little endian机器上编译代码)为little endian,则nothing.
//2:如果BYTE_ORDER(在big endian机器上编译代码)为big endian,则将zltail field的值由little endian转换为big endian.
#define ZIPLIST_ENTRY_TAIL(zl) ((zl)+intrev32ifbe(ZIPLIST_TAIL_OFFSET(zl)))
/* Return the pointer to the last byte of a ziplist, which is, the
* end of ziplist FF entry. */
#define ZIPLIST_ENTRY_END(zl) ((zl)+intrev32ifbe(ZIPLIST_BYTES(zl))-1)
/* Increment the number of items field in the ziplist header. Note that this
* macro should never overflow the unsigned 16 bit integer, since entires are
* always pushed one at a time. When UINT16_MAX is reached we want the count
* to stay there to signal that a full scan is needed to get the number of
* items inside the ziplist. */
//incr ziplist中entry的数量
#define ZIPLIST_INCR_LENGTH(zl,incr) { \
if (ZIPLIST_LENGTH(zl) < UINT16_MAX) \
ZIPLIST_LENGTH(zl) = intrev16ifbe(intrev16ifbe(ZIPLIST_LENGTH(zl))+incr); \
}
/* We use this function to receive information about a ziplist entry.
* Note that this is not how the data is actually encoded, is just what we
* get filled by a function in order to operate more easily. */
typedef struct zlentry {
//表示prelen字段的长度,1or5bytes
unsigned int prevrawlensize; /* Bytes used to encode the previos entry len*/
//用于表示prev entry的长度
unsigned int prevrawlen; /* Previous entry len. */
//用于表示encoding字段的长度
unsigned int lensize; /* Bytes used to encode this entry type/len.
For example strings have a 1, 2 or 5 bytes
header. Integers always use a single byte.*/
//content字段的长度
unsigned int len; /* Bytes used to represent the actual entry.
For strings this is just the string length
while for integers it is 1, 2, 3, 4, 8 or
0 (for 4 bit immediate) depending on the
number range. */
//prelen+encoding字段的长度
unsigned int headersize; /* prevrawlensize + lensize. */
//表示是string编码还是integer编码,长度为固定值1bytes
unsigned char encoding; /* Set to ZIP_STR_* or ZIP_INT_* depending on
the entry encoding. However for 4 bits
immediate integers this can assume a range
of values and must be range-checked. */
//用于指向当前entry的首地址
unsigned char *p; /* Pointer to the very start of the entry, that
is, this points to prev-entry-len field. */
} zlentry;
//用于初始化zlentry结构体
#define ZIPLIST_ENTRY_ZERO(zle) { \
(zle)->prevrawlensize = (zle)->prevrawlen = 0; \
(zle)->lensize = (zle)->len = (zle)->headersize = 0; \
(zle)->encoding = 0; \
(zle)->p = NULL; \
}
/* Extract the encoding from the byte pointed by 'ptr' and set it into
* 'encoding' field of the zlentry structure. */
//从ptr指向的内存中获取zlentry中的encoding字段,如果是integer则正好是1bytes,即ptr[0];
//如果是string类型,需要将encoding中表示content的长度部分清0
#define ZIP_ENTRY_ENCODING(ptr, encoding) do { \
(encoding) = (ptr[0]); \
if ((encoding) < ZIP_STR_MASK) (encoding) &= ZIP_STR_MASK; \
} while(0)
/* Return bytes needed to store integer encoded by 'encoding'. */
//根据encoding,获取content的长度
unsigned int zipIntSize(unsigned char encoding) {
switch(encoding) {
case ZIP_INT_8B: return 1;
case ZIP_INT_16B: return 2;
case ZIP_INT_24B: return 3;
case ZIP_INT_32B: return 4;
case ZIP_INT_64B: return 8;
}
if (encoding >= ZIP_INT_IMM_MIN && encoding <= ZIP_INT_IMM_MAX)
return 0; /* 4 bit immediate */
panic("Invalid integer encoding 0x%02X", encoding);
return 0;
}
/* Write the encoidng header of the entry in 'p'. If p is NULL it just returns
* the amount of bytes required to encode such a length. Arguments:
*
* 'encoding' is the encoding we are using for the entry. It could be
* ZIP_INT_* or ZIP_STR_* or between ZIP_INT_IMM_MIN and ZIP_INT_IMM_MAX
* for single-byte small immediate integers.
*
* 'rawlen' is only used for ZIP_STR_* encodings and is the length of the
* srting that this entry represents.
*
* The function returns the number of bytes used by the encoding/length
* header stored in 'p'. */
//根据参数encoding(只判断encoding的低位2bits)判断是string还是integer
//如果是string则将type/len进行拼接encoding,并保存在p指向的内存
//如果是integer则将encoding 保存在p指向的内存
//函数返回值位encoding字段的长度
//rawlen 表示string的长度
unsigned int zipStoreEntryEncoding(unsigned char *p, unsigned char encoding, unsigned int rawlen) {
unsigned char len = 1, buf[5];
if (ZIP_IS_STR(encoding)) {
/* Although encoding is given it may not be set for strings,
* so we determine it here using the raw length. */
if (rawlen <= 0x3f) {
if (!p) return len;
buf[0] = ZIP_STR_06B | rawlen;
} else if (rawlen <= 0x3fff) {
len += 1;
if (!p) return len;
buf[0] = ZIP_STR_14B | ((rawlen >> 8) & 0x3f);
buf[1] = rawlen & 0xff;
} else {
len += 4;
if (!p) return len;
buf[0] = ZIP_STR_32B;
buf[1] = (rawlen >> 24) & 0xff;
buf[2] = (rawlen >> 16) & 0xff;
buf[3] = (rawlen >> 8) & 0xff;
buf[4] = rawlen & 0xff;
}
} else {
/* Implies integer encoding, so length is always 1. */
if (!p) return len;
buf[0] = encoding;
}
/* Store this length at p. */
memcpy(p,buf,len);
return len;
}
/* Decode the entry encoding type and data length (string length for strings,
* number of bytes used for the integer for integer entries) encoded in 'ptr'.
* The 'encoding' variable will hold the entry encoding, the 'lensize'
* variable will hold the number of bytes required to encode the entry
* length, and the 'len' variable will hold the entry length. */
//lensize表示encoding字段的长度
//len表示content的长度
#define ZIP_DECODE_LENGTH(ptr, encoding, lensize, len) do { \
ZIP_ENTRY_ENCODING((ptr), (encoding)); \
if ((encoding) < ZIP_STR_MASK) { \
if ((encoding) == ZIP_STR_06B) { \
(lensize) = 1; \
(len) = (ptr)[0] & 0x3f; \
} else if ((encoding) == ZIP_STR_14B) { \
(lensize) = 2; \
(len) = (((ptr)[0] & 0x3f) << 8) | (ptr)[1]; \
} else if ((encoding) == ZIP_STR_32B) { \
(lensize) = 5; \
(len) = ((ptr)[1] << 24) | \
((ptr)[2] << 16) | \
((ptr)[3] << 8) | \
((ptr)[4]); \
} else { \
panic("Invalid string encoding 0x%02X", (encoding)); \
} \
} else { \
(lensize) = 1; \
(len) = zipIntSize(encoding); \
} \
} while(0);
/* Encode the length of the previous entry and write it to "p". This only
* uses the larger encoding (required in __ziplistCascadeUpdate). */
//prelen字段的长度为5bytes,将len存储到prelen字段
int zipStorePrevEntryLengthLarge(unsigned char *p, unsigned int len) {
if (p != NULL) {
p[0] = ZIP_BIG_PREVLEN;
memcpy(p+1,&len,sizeof(len));
memrev32ifbe(p+1);
}
return 1+sizeof(len);
}
/* Encode the length of the previous entry and write it to "p". Return the
* number of bytes needed to encode this length if "p" is NULL. */
//获取entry的pre length field
unsigned int zipStorePrevEntryLength(unsigned char *p, unsigned int len) {
if (p == NULL) {
return (len < ZIP_BIG_PREVLEN) ? 1 : sizeof(len)+1;
} else {
if (len < ZIP_BIG_PREVLEN) {
p[0] = len;
return 1;
} else {
return zipStorePrevEntryLengthLarge(p,len);
}
}
}
/* Return the number of bytes used to encode the length of the previous
* entry. The length is returned by setting the var 'prevlensize'. */
#define ZIP_DECODE_PREVLENSIZE(ptr, prevlensize) do { \
if ((ptr)[0] < ZIP_BIG_PREVLEN) { \
(prevlensize) = 1; \
} else { \
(prevlensize) = 5; \
} \
} while(0);
/* Return the length of the previous element, and the number of bytes that
* are used in order to encode the previous element length.
* 'ptr' must point to the prevlen prefix of an entry (that encodes the
* length of the previos entry in order to navigate the elements backward).
* The length of the previous entry is stored in 'prevlen', the number of
* bytes needed to encode the previous entry length are stored in
* 'prevlensize'. */
#define ZIP_DECODE_PREVLEN(ptr, prevlensize, prevlen) do { \
ZIP_DECODE_PREVLENSIZE(ptr, prevlensize); \
if ((prevlensize) == 1) { \
(prevlen) = (ptr)[0]; \
} else if ((prevlensize) == 5) { \
assert(sizeof((prevlensize)) == 4);//assert(sizeof(prelen)==4);???? \
memcpy(&(prevlen), ((char*)(ptr)) + 1, 4); \
memrev32ifbe(&prevlen); \
} \
} while(0);
/* Given a pointer 'p' to the prevlen info that prefixes an entry, this
* function returns the difference in number of bytes needed to encode
* the prevlen if the previous entry changes of size.
*
* So if A is the number of bytes used right now to encode the 'prevlen'
* field.
*
* And B is the number of bytes that are needed in order to encode the
* 'prevlen' if the previous element will be updated to one of size 'len'.
*
* Then the function returns B - A
*
* So the function returns a positive number if more space is needed,
* a negative number if less space is needed, or zero if the same space
* is needed. */
//返回new prelensize - old prevlensize
int zipPrevLenByteDiff(unsigned char *p, unsigned int len) {
unsigned int prevlensize;
ZIP_DECODE_PREVLENSIZE(p, prevlensize);
return zipStorePrevEntryLength(NULL, len) - prevlensize;
}
/* Return the total number of bytes used by the entry pointed to by 'p'. */
// 获取entry total length
unsigned int zipRawEntryLength(unsigned char *p) {
unsigned int prevlensize, encoding, lensize, len;
ZIP_DECODE_PREVLENSIZE(p, prevlensize);
ZIP_DECODE_LENGTH(p + prevlensize, encoding, lensize, len);
return prevlensize + lensize + len;
}
/* Check if string pointed to by 'entry' can be encoded as an integer.
* Stores the integer value in 'v' and its encoding in 'encoding'. */
int zipTryEncoding(unsigned char *entry, unsigned int entrylen, long long *v, unsigned char *encoding) {
long long value;
if (entrylen >= 32 || entrylen == 0) return 0;
if (string2ll((char*)entry,entrylen,&value)) {
/* Great, the string can be encoded. Check what's the smallest
* of our encoding types that can hold this value. */
//immediate 4 bits integer
if (value >= 0 && value <= 12) {
*encoding = ZIP_INT_IMM_MIN+value;
} else if (value >= INT8_MIN && value <= INT8_MAX) {
*encoding = ZIP_INT_8B;
} else if (value >= INT16_MIN && value <= INT16_MAX) {
*encoding = ZIP_INT_16B;
} else if (value >= INT24_MIN && value <= INT24_MAX) {
*encoding = ZIP_INT_24B;
} else if (value >= INT32_MIN && value <= INT32_MAX) {
*encoding = ZIP_INT_32B;
} else {
*encoding = ZIP_INT_64B;
}
*v = value;
return 1;
}
return 0;
}
/* Store integer 'value' at 'p', encoded as 'encoding' */
void zipSaveInteger(unsigned char *p, int64_t value, unsigned char encoding) {
int16_t i16;
int32_t i32;
int64_t i64;
if (encoding == ZIP_INT_8B) {
((int8_t*)p)[0] = (int8_t)value;
} else if (encoding == ZIP_INT_16B) {
i16 = value;
memcpy(p,&i16,sizeof(i16));
memrev16ifbe(p);
} else if (encoding == ZIP_INT_24B) {
i32 = value<<8;
memrev32ifbe(&i32);
memcpy(p,((uint8_t*)&i32)+1,sizeof(i32)-sizeof(uint8_t));
} else if (encoding == ZIP_INT_32B) {
i32 = value;
memcpy(p,&i32,sizeof(i32));
memrev32ifbe(p);
} else if (encoding == ZIP_INT_64B) {
i64 = value;
memcpy(p,&i64,sizeof(i64));
memrev64ifbe(p);
} else if (encoding >= ZIP_INT_IMM_MIN && encoding <= ZIP_INT_IMM_MAX) {
/* Nothing to do, the value is stored in the encoding itself. */
} else {
assert(NULL);
}
}
/* Read integer encoded as 'encoding' from 'p' */
//获取int_*的content
int64_t zipLoadInteger(unsigned char *p, unsigned char encoding) {
int16_t i16;
int32_t i32;
int64_t i64, ret = 0;
if (encoding == ZIP_INT_8B) {
ret = ((int8_t*)p)[0];
} else if (encoding == ZIP_INT_16B) {
memcpy(&i16,p,sizeof(i16));
memrev16ifbe(&i16);
ret = i16;
} else if (encoding == ZIP_INT_32B) {
memcpy(&i32,p,sizeof(i32));
memrev32ifbe(&i32);
ret = i32;
} else if (encoding == ZIP_INT_24B) {
i32 = 0;
memcpy(((uint8_t*)&i32)+1,p,sizeof(i32)-sizeof(uint8_t));
memrev32ifbe(&i32);
ret = i32>>8;
} else if (encoding == ZIP_INT_64B) {
memcpy(&i64,p,sizeof(i64));
memrev64ifbe(&i64);
ret = i64;
} else if (encoding >= ZIP_INT_IMM_MIN && encoding <= ZIP_INT_IMM_MAX) {
ret = (encoding & ZIP_INT_IMM_MASK)-1;
} else {
assert(NULL);
}
return ret;
}
/* Return a struct with all information about an entry. */
//根据p指向的内存,获取zlentry结构体
void zipEntry(unsigned char *p, zlentry *e) {
//获取prevlensize(prelen字段的长度)和prelen字段
ZIP_DECODE_PREVLEN(p, e->prevrawlensize, e->prevrawlen);
//获取encoding,lensize(encodiing字段的长度),和len(content的长度)
ZIP_DECODE_LENGTH(p + e->prevrawlensize, e->encoding, e->lensize, e->len);
//prevlen 和encoding字段的长度
e->headersize = e->prevrawlensize + e->lensize;
//指向当前entry的首地址
e->p = p;
}
/* Create a new empty ziplist. */
/*创建一个empty的ziplist
*
* |-------|------------|-----|
* |zlbytes|zltail|zllen|zlend|
* |-------|------------|-----|
*
*/
unsigned char *ziplistNew(void) {
unsigned int bytes = ZIPLIST_HEADER_SIZE+1;
unsigned char *zl = zmalloc(bytes);
ZIPLIST_BYTES(zl) = intrev32ifbe(bytes);
ZIPLIST_TAIL_OFFSET(zl) = intrev32ifbe(ZIPLIST_HEADER_SIZE);
ZIPLIST_LENGTH(zl) = 0;
zl[bytes-1] = ZIP_END;
return zl;
}
/* Resize the ziplist. */
//realloc ziplist
unsigned char *ziplistResize(unsigned char *zl, unsigned int len) {
zl = zrealloc(zl,len);
//total bytes
ZIPLIST_BYTES(zl) = intrev32ifbe(len);
zl[len-1] = ZIP_END;
return zl;
}
/* When an entry is inserted, we need to set the prevlen field of the next
* entry to equal the length of the inserted entry. It can occur that this
* length cannot be encoded in 1 byte and the next entry needs to be grow
* a bit larger to hold the 5-byte encoded prevlen. This can be done for free,
* because this only happens when an entry is already being inserted (which
* causes a realloc and memmove). However, encoding the prevlen may require
* that this entry is grown as well. This effect may cascade throughout
* the ziplist when there are consecutive entries with a size close to
* ZIP_BIG_PREVLEN, so we need to check that the prevlen can be encoded in
* every consecutive entry.
*
* Note that this effect can also happen in reverse, where the bytes required
* to encode the prevlen field can shrink. This effect is deliberately ignored,
* because it can cause a "flapping" effect where a chain prevlen fields is
* first grown and then shrunk again after consecutive inserts. Rather, the
* field is allowed to stay larger than necessary, because a large prevlen
* field implies the ziplist is holding large entries anyway.
*
* The pointer "p" points to the first entry that does NOT need to be
* updated, i.e. consecutive fields MAY need an update. */
//连锁更新
unsigned char *__ziplistCascadeUpdate(unsigned char *zl, unsigned char *p) {
//curlen表示zl的total len
size_t curlen = intrev32ifbe(ZIPLIST_BYTES(zl)), rawlen, rawlensize;
size_t offset, noffset, extra;
unsigned char *np;
zlentry cur, next;
//p[0]为first entry的首地址
while (p[0] != ZIP_END) {
zipEntry(p, &cur);
//first entry的total length
rawlen = cur.headersize + cur.len;
//获取next entry的prevlen field的需要多少个bytes(1 or 5)来存储更新的first entry的total length
rawlensize = zipStorePrevEntryLength(NULL,rawlen);
/* Abort if there is no next entry. */
//当前engry没有nextentry.
if (p[rawlen] == ZIP_END) break;
//获取next entry的zlentry结构体
zipEntry(p+rawlen, &next);
/* Abort when "prevlen" has not changed. */
if (next.prevrawlen == rawlen) break;
//next entry的prevlen字段需要expand
if (next.prevrawlensize < rawlensize) {
/* The "prevlen" field of "next" needs more bytes to hold
* the raw length of "cur". */
//first entry的首地址相对于zp首地址的偏移量
offset = p-zl;
//next entry prevlen字段需要扩展的字节数
extra = rawlensize-next.prevrawlensize;
//realloc zp
zl = ziplistResize(zl,curlen+extra);
//first entry的首地址
p = zl+offset;
/* Current pointer and offset for next element. */
//rawlen为first entry的total length , np means next entry的首地址
np = p+rawlen;
//next entry相对于zp首地址的偏移量
noffset = np-zl;
/* Update tail offset when next element is not the tail element. */
//next entry非ziplist 的last entry
if ((zl+intrev32ifbe(ZIPLIST_TAIL_OFFSET(zl))) != np) {
ZIPLIST_TAIL_OFFSET(zl) =
intrev32ifbe(intrev32ifbe(ZIPLIST_TAIL_OFFSET(zl))+extra);
}
/* Move the tail to the back. */
memmove(np+rawlensize,
np+next.prevrawlensize,
curlen-noffset-next.prevrawlensize-1);
//存储next entry的prelen
zipStorePrevEntryLength(np,rawlen);
/* Advance the cursor */
p += rawlen;
curlen += extra;
} else {
if (next.prevrawlensize > rawlensize) {
/* This would result in shrinking, which we want to avoid.
* So, set "rawlen" in the available bytes. */
zipStorePrevEntryLengthLarge(p+rawlen,rawlen);
} else {
zipStorePrevEntryLength(p+rawlen,rawlen);
}
/* Stop here, as the raw length of "next" has not changed. */
break;
}
}
return zl;
}
/* Delete "num" entries, starting at "p". Returns pointer to the ziplist. */
//删除num个entry
unsigned char *__ziplistDelete(unsigned char *zl, unsigned char *p, unsigned int num) {
unsigned int i, totlen, deleted = 0;
size_t offset;
int nextdiff = 0;
zlentry first, tail;
zipEntry(p, &first);
for (i = 0; p[0] != ZIP_END && i < num; i++) {
p += zipRawEntryLength(p);
deleted++;
}
totlen = p-first.p; /* Bytes taken by the element(s) to delete. */
if (totlen > 0) {
//p not zlend
if (p[0] != ZIP_END) {
/* Storing `prevrawlen` in this entry may increase or decrease the
* number of bytes required compare to the current `prevrawlen`.
* There always is room to store this, because it was previously
* stored by an entry that is now being deleted. */
//nextdiff = first prevlensize 减去 第num+1个entry的prevlensize的差值
nextdiff = zipPrevLenByteDiff(p,first.prevrawlen);
/* Note that there is always space when p jumps backward: if
* the new previous entry is large, one of the deleted elements
* had a 5 bytes prevlen header, so there is for sure at least
* 5 bytes free and we need just 4. */
p -= nextdiff;
//存储first entry前边的entry(即第0个entry)的length
zipStorePrevEntryLength(p,first.prevrawlen);
/* Update offset for tail */
ZIPLIST_TAIL_OFFSET(zl) =
intrev32ifbe(intrev32ifbe(ZIPLIST_TAIL_OFFSET(zl))-totlen);
/* When the tail contains more than one entry, we need to take
* "nextdiff" in account as well. Otherwise, a change in the
* size of prevlen doesn't have an effect on the *tail* offset. */
zipEntry(p, &tail);
if (p[tail.headersize+tail.len] != ZIP_END) {
ZIPLIST_TAIL_OFFSET(zl) =
intrev32ifbe(intrev32ifbe(ZIPLIST_TAIL_OFFSET(zl))+nextdiff);
}
/* Move tail to the front of the ziplist */
memmove(first.p,p,
intrev32ifbe(ZIPLIST_BYTES(zl))-(p-zl)-1);
} else {
/* The entire tail was deleted. No need to move memory. */
//删除[first, last]entry之间的所有entry(包括last entry)
ZIPLIST_TAIL_OFFSET(zl) =
intrev32ifbe((first.p-zl)-first.prevrawlen);
}
/* Resize and update length */
offset = first.p-zl;
zl = ziplistResize(zl, intrev32ifbe(ZIPLIST_BYTES(zl))-totlen+nextdiff);
//更新engry数量
ZIPLIST_INCR_LENGTH(zl,-deleted);
p = zl+offset;
/* When nextdiff != 0, the raw length of the next entry has changed, so
* we need to cascade the update throughout the ziplist */
//可能需要连锁更新
if (nextdiff != 0)
zl = __ziplistCascadeUpdate(zl,p);
}
return zl;
}
/* Insert item at "p". */
unsigned char *__ziplistInsert(unsigned char *zl, unsigned char *p, unsigned char *s, unsigned int slen) {
size_t curlen = intrev32ifbe(ZIPLIST_BYTES(zl)), reqlen;
unsigned int prevlensize, prevlen = 0;
size_t offset;
int nextdiff = 0;
unsigned char encoding = 0;
long long value = 123456789; /* initialized to avoid warning. Using a value
that is easy to see if for some reason
we use it uninitialized. */
zlentry tail;
/* Find out prevlen for the entry that is inserted. */
if (p[0] != ZIP_END) {
//获得p指向entry的prelensize和前一个entry的total length
ZIP_DECODE_PREVLEN(p, prevlensize, prevlen);
} else {
//如果p指向zllend
unsigned char *ptail = ZIPLIST_ENTRY_TAIL(zl);
if (ptail[0] != ZIP_END) {
prevlen = zipRawEntryLength(ptail);
}
}
/* See if the entry can be encoded */
//reqlen为entry content的长度
if (zipTryEncoding(s,slen,&value,&encoding)) {
/* 'encoding' is set to the appropriate integer encoding */
//integer的content长度
reqlen = zipIntSize(encoding);
} else {
/* 'encoding' is untouched, however zipStoreEntryEncoding will use the
* string length to figure out how to encode it. */
//string ,slen为string的长度
reqlen = slen;
}
/* We need space for both the length of the previous entry and
* the length of the payload. */
//获取prevlensize
reqlen += zipStorePrevEntryLength(NULL,prevlen);
//获取entry的encoding字段长度
reqlen += zipStoreEntryEncoding(NULL,encoding,slen);
/* When the insert position is not equal to the tail, we need to
* make sure that the next entry can hold this entry's length in
* its prevlen field. */
int forcelarge = 0;
nextdiff = (p[0] != ZIP_END) ? zipPrevLenByteDiff(p,reqlen) : 0;
//ziplist的total length不能变小
if (nextdiff == -4 && reqlen < 4) {
nextdiff = 0;
forcelarge = 1;
}
/* Store offset because a realloc may change the address of zl. */
offset = p-zl;
zl = ziplistResize(zl,curlen+reqlen+nextdiff);
p = zl+offset;
/* Apply memory move when necessary and update tail offset. */
if (p[0] != ZIP_END) {
/* Subtract one because of the ZIP_END bytes */
memmove(p+reqlen,p-nextdiff,curlen-offset-1+nextdiff);
/* Encode this entry's raw length in the next entry. */
//forcelarge==1强制采用5bytes存储prevlen field
if (forcelarge)
zipStorePrevEntryLengthLarge(p+reqlen,reqlen);
else
zipStorePrevEntryLength(p+reqlen,reqlen);
/* Update offset for tail */
ZIPLIST_TAIL_OFFSET(zl) =
intrev32ifbe(intrev32ifbe(ZIPLIST_TAIL_OFFSET(zl))+reqlen);
/* When the tail contains more than one entry, we need to take
* "nextdiff" in account as well. Otherwise, a change in the
* size of prevlen doesn't have an effect on the *tail* offset. */
zipEntry(p+reqlen, &tail);
if (p[reqlen+tail.headersize+tail.len] != ZIP_END) {
ZIPLIST_TAIL_OFFSET(zl) =
intrev32ifbe(intrev32ifbe(ZIPLIST_TAIL_OFFSET(zl))+nextdiff);
}
} else {
/* This element will be the new tail. */
ZIPLIST_TAIL_OFFSET(zl) = intrev32ifbe(p-zl);
}
/* When nextdiff != 0, the raw length of the next entry has changed, so
* we need to cascade the update throughout the ziplist */
if (nextdiff != 0) {
offset = p-zl;
zl = __ziplistCascadeUpdate(zl,p+reqlen);
p = zl+offset;
}
/* Write the entry */
p += zipStorePrevEntryLength(p,prevlen);
p += zipStoreEntryEncoding(p,encoding,slen);
if (ZIP_IS_STR(encoding)) {
memcpy(p,s,slen);
} else {
zipSaveInteger(p,value,encoding);
}
ZIPLIST_INCR_LENGTH(zl,1);
return zl;
}
/* Merge ziplists 'first' and 'second' by appending 'second' to 'first'.
*
* NOTE: The larger ziplist is reallocated to contain the new merged ziplist.
* Either 'first' or 'second' can be used for the result. The parameter not
* used will be free'd and set to NULL.
*
* After calling this function, the input parameters are no longer valid since
* they are changed and free'd in-place.
*
* The result ziplist is the contents of 'first' followed by 'second'.
*
* On failure: returns NULL if the merge is impossible.
* On success: returns the merged ziplist (which is expanded version of either
* 'first' or 'second', also frees the other unused input ziplist, and sets the
* input ziplist argument equal to newly reallocated ziplist return value. */
unsigned char *ziplistMerge(unsigned char **first, unsigned char **second) {
/* If any params are null, we can't merge, so NULL. */
if (first == NULL || *first == NULL || second == NULL || *second == NULL)
return NULL;
/* Can't merge same list into itself. */
//如果*first指向的ziplist和*second指向的ziplist相同
if (*first == *second)
return NULL;
//zl total bytes
size_t first_bytes = intrev32ifbe(ZIPLIST_BYTES(*first));
//zl totol entry number
size_t first_len = intrev16ifbe(ZIPLIST_LENGTH(*first));
size_t second_bytes = intrev32ifbe(ZIPLIST_BYTES(*second));
size_t second_len = intrev16ifbe(ZIPLIST_LENGTH(*second));
int append;
unsigned char *source, *target;
size_t target_bytes, source_bytes;
/* Pick the largest ziplist so we can resize easily in-place.
* We must also track if we are now appending or prepending to
* the target ziplist. */
if (first_len >= second_len) {
/* retain first, append second to first. */
target = *first;
target_bytes = first_bytes;
source = *second;
source_bytes = second_bytes;
//appending *seond指向的zl追加到*first指向的entry后边
append = 1;
} else {
/* else, retain second, prepend first to second. */
target = *second;
target_bytes = second_bytes;
source = *first;
source_bytes = first_bytes;
//prepanding *first指向的zl prepend到*second指向的zl前边
append = 0;
}
/* Calculate final bytes (subtract one pair of metadata) */
//两个zl merged后的total length
size_t zlbytes = first_bytes + second_bytes -
ZIPLIST_HEADER_SIZE - ZIPLIST_END_SIZE;
//两个zl merged后的total entry number
size_t zllength = first_len + second_len;
/* Combined zl length should be limited within UINT16_MAX */
zllength = zllength < UINT16_MAX ? zllength : UINT16_MAX;
/* Save offset positions before we start ripping memory apart. */
//last entry offset
size_t first_offset = intrev32ifbe(ZIPLIST_TAIL_OFFSET(*first));
size_t second_offset = intrev32ifbe(ZIPLIST_TAIL_OFFSET(*second));
/* Extend target to new zlbytes then append or prepend source. */
target = zrealloc(target, zlbytes);
if (append) {
/* append == appending to target */
/* Copy source after target (copying over original [END]):
* [TARGET - END, SOURCE - HEADER] */
//将*second指向的zl追加到*first指向的zl后边
memcpy(target + target_bytes - ZIPLIST_END_SIZE,
source + ZIPLIST_HEADER_SIZE,
source_bytes - ZIPLIST_HEADER_SIZE);
} else {
/* !append == prepending to target */
/* Move target *contents* exactly size of (source - [END]),
* then copy source into vacataed space (source - [END]):
* [SOURCE - END, TARGET - HEADER] */
//将*first指向的zl添加到*second指向的zl前边
memmove(target + source_bytes - ZIPLIST_END_SIZE,
target + ZIPLIST_HEADER_SIZE,
target_bytes - ZIPLIST_HEADER_SIZE);
memcpy(target, source, source_bytes - ZIPLIST_END_SIZE);
}
/* Update header metadata. */
ZIPLIST_BYTES(target) = intrev32ifbe(zlbytes);
ZIPLIST_LENGTH(target) = intrev16ifbe(zllength);
/* New tail offset is:
* + N bytes of first ziplist
* - 1 byte for [END] of first ziplist
* + M bytes for the offset of the original tail of the second ziplist
* - J bytes for HEADER because second_offset keeps no header. */
ZIPLIST_TAIL_OFFSET(target) = intrev32ifbe(
(first_bytes - ZIPLIST_END_SIZE) +
(second_offset - ZIPLIST_HEADER_SIZE));
/* __ziplistCascadeUpdate just fixes the prev length values until it finds a
* correct prev length value (then it assumes the rest of the list is okay).
* We tell CascadeUpdate to start at the first ziplist's tail element to fix
* the merge seam. */
target = __ziplistCascadeUpdate(target, target+first_offset);
/* Now free and NULL out what we didn't realloc */
if (append) {
zfree(*second);
*second = NULL;
*first = target;
} else {
zfree(*first);
*first = NULL;
*second = target;
}
return target;
}
unsigned char *ziplistPush(unsigned char *zl, unsigned char *s, unsigned int slen, int where) {
unsigned char *p;
p = (where == ZIPLIST_HEAD) ? ZIPLIST_ENTRY_HEAD(zl) : ZIPLIST_ENTRY_END(zl);
return __ziplistInsert(zl,p,s,slen);
}
/* Returns an offset to use for iterating with ziplistNext. When the given
* index is negative, the list is traversed back to front. When the list
* doesn't contain an element at the provided index, NULL is returned. */
unsigned char *ziplistIndex(unsigned char *zl, int index) {
unsigned char *p;
unsigned int prevlensize, prevlen = 0;
if (index < 0) {
index = (-index)-1;
p = ZIPLIST_ENTRY_TAIL(zl);
if (p[0] != ZIP_END) {
ZIP_DECODE_PREVLEN(p, prevlensize, prevlen);
while (prevlen > 0 && index--) {
p -= prevlen;
ZIP_DECODE_PREVLEN(p, prevlensize, prevlen);
}
}
} else {
p = ZIPLIST_ENTRY_HEAD(zl);
while (p[0] != ZIP_END && index--) {
p += zipRawEntryLength(p);
}
}
return (p[0] == ZIP_END || index > 0) ? NULL : p;
}
/* Return pointer to next entry in ziplist.
*
* zl is the pointer to the ziplist
* p is the pointer to the current element
*
* The element after 'p' is returned, otherwise NULL if we are at the end. */
unsigned char *ziplistNext(unsigned char *zl, unsigned char *p) {
((void) zl);
/* "p" could be equal to ZIP_END, caused by ziplistDelete,
* and we should return NULL. Otherwise, we should return NULL
* when the *next* element is ZIP_END (there is no next entry). */
if (p[0] == ZIP_END) {
return NULL;
}
p += zipRawEntryLength(p);
if (p[0] == ZIP_END) {
return NULL;
}
return p;
}
/* Return pointer to previous entry in ziplist. */
unsigned char *ziplistPrev(unsigned char *zl, unsigned char *p) {
unsigned int prevlensize, prevlen = 0;
/* Iterating backwards from ZIP_END should return the tail. When "p" is
* equal to the first element of the list, we're already at the head,
* and should return NULL. */
if (p[0] == ZIP_END) {
p = ZIPLIST_ENTRY_TAIL(zl);
return (p[0] == ZIP_END) ? NULL : p;
} else if (p == ZIPLIST_ENTRY_HEAD(zl)) {
return NULL;
} else {
ZIP_DECODE_PREVLEN(p, prevlensize, prevlen);
assert(prevlen > 0);
return p-prevlen;
}
}
/* Get entry pointed to by 'p' and store in either '*sstr' or 'sval' depending
* on the encoding of the entry. '*sstr' is always set to NULL to be able
* to find out whether the string pointer or the integer value was set.
* Return 0 if 'p' points to the end of the ziplist, 1 otherwise. */
unsigned int ziplistGet(unsigned char *p, unsigned char **sstr, unsigned int *slen, long long *sval) {
zlentry entry;
if (p == NULL || p[0] == ZIP_END) return 0;
if (sstr) *sstr = NULL;
zipEntry(p, &entry);
if (ZIP_IS_STR(entry.encoding)) {
if (sstr) {
*slen = entry.len;
*sstr = p+entry.headersize;
}
} else {
if (sval) {
*sval = zipLoadInteger(p+entry.headersize,entry.encoding);
}
}
return 1;
}
/* Insert an entry at "p". */
unsigned char *ziplistInsert(unsigned char *zl, unsigned char *p, unsigned char *s, unsigned int slen) {
return __ziplistInsert(zl,p,s,slen);
}
/* Delete a single entry from the ziplist, pointed to by *p.
* Also update *p in place, to be able to iterate over the
* ziplist, while deleting entries. */
unsigned char *ziplistDelete(unsigned char *zl, unsigned char **p) {
size_t offset = *p-zl;
zl = __ziplistDelete(zl,*p,1);
/* Store pointer to current element in p, because ziplistDelete will
* do a realloc which might result in a different "zl"-pointer.
* When the delete direction is back to front, we might delete the last
* entry and end up with "p" pointing to ZIP_END, so check this. */
*p = zl+offset;
return zl;
}
/* Delete a range of entries from the ziplist. */
unsigned char *ziplistDeleteRange(unsigned char *zl, int index, unsigned int num) {
unsigned char *p = ziplistIndex(zl,index);
return (p == NULL) ? zl : __ziplistDelete(zl,p,num);
}
/* Compare entry pointer to by 'p' with 'sstr' of length 'slen'. */
/* Return 1 if equal. */
unsigned int ziplistCompare(unsigned char *p, unsigned char *sstr, unsigned int slen) {
zlentry entry;
unsigned char sencoding;
long long zval, sval;
if (p[0] == ZIP_END) return 0;
zipEntry(p, &entry);
if (ZIP_IS_STR(entry.encoding)) {
/* Raw compare */
if (entry.len == slen) {
return memcmp(p+entry.headersize,sstr,slen) == 0;
} else {
return 0;
}
} else {
/* Try to compare encoded values. Don't compare encoding because
* different implementations may encoded integers differently. */
if (zipTryEncoding(sstr,slen,&sval,&sencoding)) {
zval = zipLoadInteger(p+entry.headersize,entry.encoding);
return zval == sval;
}
}
return 0;
}
/* Find pointer to the entry equal to the specified entry. Skip 'skip' entries
* between every comparison. Returns NULL when the field could not be found. */
unsigned char *ziplistFind(unsigned char *p, unsigned char *vstr, unsigned int vlen, unsigned int skip) {
int skipcnt = 0;
unsigned char vencoding = 0;
long long vll = 0;
while (p[0] != ZIP_END) {
unsigned int prevlensize, encoding, lensize, len;
unsigned char *q;
//获取p指向的entry的prevlensize
ZIP_DECODE_PREVLENSIZE(p, prevlensize);
//获取p指向entry的encoding的长度lensize,和content的长度len
ZIP_DECODE_LENGTH(p + prevlensize, encoding, lensize, len);
//q指向entry的content
q = p + prevlensize + lensize;
if (skipcnt == 0) {
/* Compare current entry with specified entry */
if (ZIP_IS_STR(encoding)) {
if (len == vlen && memcmp(q, vstr, vlen) == 0) {
return p;
}
} else {
/* Find out if the searched field can be encoded. Note that
* we do it only the first time, once done vencoding is set
* to non-zero and vll is set to the integer value. */
if (vencoding == 0) {
if (!zipTryEncoding(vstr, vlen, &vll, &vencoding)) {
/* If the entry can't be encoded we set it to
* UCHAR_MAX so that we don't retry again the next
* time. */
vencoding = UCHAR_MAX;
}
/* Must be non-zero by now */
assert(vencoding);
}
/* Compare current entry with specified entry, do it only
* if vencoding != UCHAR_MAX because if there is no encoding
* possible for the field it can't be a valid integer. */
if (vencoding != UCHAR_MAX) {
long long ll = zipLoadInteger(q, encoding);
if (ll == vll) {
return p;
}
}
}
/* Reset skip count */
skipcnt = skip;
} else {
/* Skip entry */
skipcnt--;
}
/* Move to next entry */
p = q + len;
}
return NULL;
}
/* Return length of ziplist. */
unsigned int ziplistLen(unsigned char *zl) {
unsigned int len = 0;
if (intrev16ifbe(ZIPLIST_LENGTH(zl)) < UINT16_MAX) {
len = intrev16ifbe(ZIPLIST_LENGTH(zl));
} else {
unsigned char *p = zl+ZIPLIST_HEADER_SIZE;
while (*p != ZIP_END) {
p += zipRawEntryLength(p);
len++;
}
/* Re-store length if small enough */
if (len < UINT16_MAX) ZIPLIST_LENGTH(zl) = intrev16ifbe(len);
}
return len;
}
/* Return ziplist blob size in bytes. */
size_t ziplistBlobLen(unsigned char *zl) {
return intrev32ifbe(ZIPLIST_BYTES(zl));
}
void ziplistRepr(unsigned char *zl) {
unsigned char *p;
int index = 0;
zlentry entry;
printf(
"{total bytes %d} "
"{num entries %u}\n"
"{tail offset %u}\n",
intrev32ifbe(ZIPLIST_BYTES(zl)),
intrev16ifbe(ZIPLIST_LENGTH(zl)),
intrev32ifbe(ZIPLIST_TAIL_OFFSET(zl)));
p = ZIPLIST_ENTRY_HEAD(zl);
while(*p != ZIP_END) {
zipEntry(p, &entry);
printf(
"{\n"
"\taddr 0x%08lx,\n"
"\tindex %2d,\n"
"\toffset %5ld,\n"
"\thdr+entry len: %5u,\n"
"\thdr len%2u,\n"
"\tprevrawlen: %5u,\n"
"\tprevrawlensize: %2u,\n"
"\tpayload %5u\n",
(long unsigned)p,
index,
(unsigned long) (p-zl),
entry.headersize+entry.len,
entry.headersize,
entry.prevrawlen,
entry.prevrawlensize,
entry.len);
printf("\tbytes: ");
for (unsigned int i = 0; i < entry.headersize+entry.len; i++) {
printf("%02x|",p[i]);
}
printf("\n");
p += entry.headersize;
if (ZIP_IS_STR(entry.encoding)) {
printf("\t[str]");
if (entry.len > 40) {
if (fwrite(p,40,1,stdout) == 0) perror("fwrite");
printf("...");
} else {
if (entry.len &&
fwrite(p,entry.len,1,stdout) == 0) perror("fwrite");
}
} else {
printf("\t[int]%lld", (long long) zipLoadInteger(p,entry.encoding));
}
printf("\n}\n");
p += entry.len;
index++;
}
printf("{end}\n\n");
}
#ifdef REDIS_TEST
#include <sys/time.h>
#include "adlist.h"
#include "sds.h"
#define debug(f, ...) { if (DEBUG) printf(f, __VA_ARGS__); }
static unsigned char *createList() {
unsigned char *zl = ziplistNew();
zl = ziplistPush(zl, (unsigned char*)"foo", 3, ZIPLIST_TAIL);
zl = ziplistPush(zl, (unsigned char*)"quux", 4, ZIPLIST_TAIL);
zl = ziplistPush(zl, (unsigned char*)"hello", 5, ZIPLIST_HEAD);
zl = ziplistPush(zl, (unsigned char*)"1024", 4, ZIPLIST_TAIL);
return zl;
}
static unsigned char *createIntList() {
unsigned char *zl = ziplistNew();
char buf[32];
sprintf(buf, "100");
zl = ziplistPush(zl, (unsigned char*)buf, strlen(buf), ZIPLIST_TAIL);
sprintf(buf, "128000");
zl = ziplistPush(zl, (unsigned char*)buf, strlen(buf), ZIPLIST_TAIL);
sprintf(buf, "-100");
zl = ziplistPush(zl, (unsigned char*)buf, strlen(buf), ZIPLIST_HEAD);
sprintf(buf, "4294967296");
zl = ziplistPush(zl, (unsigned char*)buf, strlen(buf), ZIPLIST_HEAD);
sprintf(buf, "non integer");
zl = ziplistPush(zl, (unsigned char*)buf, strlen(buf), ZIPLIST_TAIL);
sprintf(buf, "much much longer non integer");
zl = ziplistPush(zl, (unsigned char*)buf, strlen(buf), ZIPLIST_TAIL);
return zl;
}
static long long usec(void) {
struct timeval tv;
gettimeofday(&tv,NULL);
return (((long long)tv.tv_sec)*1000000)+tv.tv_usec;
}
static void stress(int pos, int num, int maxsize, int dnum) {
int i,j,k;
unsigned char *zl;
char posstr[2][5] = { "HEAD", "TAIL" };
long long start;
for (i = 0; i < maxsize; i+=dnum) {
zl = ziplistNew();
for (j = 0; j < i; j++) {
zl = ziplistPush(zl,(unsigned char*)"quux",4,ZIPLIST_TAIL);
}
/* Do num times a push+pop from pos */
start = usec();
for (k = 0; k < num; k++) {
zl = ziplistPush(zl,(unsigned char*)"quux",4,pos);
zl = ziplistDeleteRange(zl,0,1);
}
printf("List size: %8d, bytes: %8d, %dx push+pop (%s): %6lld usec\n",
i,intrev32ifbe(ZIPLIST_BYTES(zl)),num,posstr[pos],usec()-start);
zfree(zl);
}
}
static unsigned char *pop(unsigned char *zl, int where) {
unsigned char *p, *vstr;
unsigned int vlen;
long long vlong;
p = ziplistIndex(zl,where == ZIPLIST_HEAD ? 0 : -1);
if (ziplistGet(p,&vstr,&vlen,&vlong)) {
if (where == ZIPLIST_HEAD)
printf("Pop head: ");
else
printf("Pop tail: ");
if (vstr) {
if (vlen && fwrite(vstr,vlen,1,stdout) == 0) perror("fwrite");
}
else {
printf("%lld", vlong);
}
printf("\n");
return ziplistDelete(zl,&p);
} else {
printf("ERROR: Could not pop\n");
exit(1);
}
}
static int randstring(char *target, unsigned int min, unsigned int max) {
int p = 0;
int len = min+rand()%(max-min+1);
int minval, maxval;
switch(rand() % 3) {
case 0:
minval = 0;
maxval = 255;
break;
case 1:
minval = 48;
maxval = 122;
break;
case 2:
minval = 48;
maxval = 52;
break;
default:
assert(NULL);
}
while(p < len)
target[p++] = minval+rand()%(maxval-minval+1);
return len;
}
static void verify(unsigned char *zl, zlentry *e) {
int len = ziplistLen(zl);
zlentry _e;
ZIPLIST_ENTRY_ZERO(&_e);
for (int i = 0; i < len; i++) {
memset(&e[i], 0, sizeof(zlentry));
zipEntry(ziplistIndex(zl, i), &e[i]);
memset(&_e, 0, sizeof(zlentry));
zipEntry(ziplistIndex(zl, -len+i), &_e);
assert(memcmp(&e[i], &_e, sizeof(zlentry)) == 0);
}
}
int ziplistTest(int argc, char **argv) {
unsigned char *zl, *p;
unsigned char *entry;
unsigned int elen;
long long value;
/* If an argument is given, use it as the random seed. */
if (argc == 2)
srand(atoi(argv[1]));
zl = createIntList();
ziplistRepr(zl);
zfree(zl);
zl = createList();
ziplistRepr(zl);
zl = pop(zl,ZIPLIST_TAIL);
ziplistRepr(zl);
zl = pop(zl,ZIPLIST_HEAD);
ziplistRepr(zl);
zl = pop(zl,ZIPLIST_TAIL);
ziplistRepr(zl);
zl = pop(zl,ZIPLIST_TAIL);
ziplistRepr(zl);
zfree(zl);
printf("Get element at index 3:\n");
{
zl = createList();
p = ziplistIndex(zl, 3);
if (!ziplistGet(p, &entry, &elen, &value)) {
printf("ERROR: Could not access index 3\n");
return 1;
}
if (entry) {
if (elen && fwrite(entry,elen,1,stdout) == 0) perror("fwrite");
printf("\n");
} else {
printf("%lld\n", value);
}
printf("\n");
zfree(zl);
}
printf("Get element at index 4 (out of range):\n");
{
zl = createList();
p = ziplistIndex(zl, 4);
if (p == NULL) {
printf("No entry\n");
} else {
printf("ERROR: Out of range index should return NULL, returned offset: %ld\n", p-zl);
return 1;
}
printf("\n");
zfree(zl);
}
printf("Get element at index -1 (last element):\n");
{
zl = createList();
p = ziplistIndex(zl, -1);
if (!ziplistGet(p, &entry, &elen, &value)) {
printf("ERROR: Could not access index -1\n");
return 1;
}
if (entry) {
if (elen && fwrite(entry,elen,1,stdout) == 0) perror("fwrite");
printf("\n");
} else {
printf("%lld\n", value);
}
printf("\n");
zfree(zl);
}
printf("Get element at index -4 (first element):\n");
{
zl = createList();
p = ziplistIndex(zl, -4);
if (!ziplistGet(p, &entry, &elen, &value)) {
printf("ERROR: Could not access index -4\n");
return 1;
}
if (entry) {
if (elen && fwrite(entry,elen,1,stdout) == 0) perror("fwrite");
printf("\n");
} else {
printf("%lld\n", value);
}
printf("\n");
zfree(zl);
}
printf("Get element at index -5 (reverse out of range):\n");
{
zl = createList();
p = ziplistIndex(zl, -5);
if (p == NULL) {
printf("No entry\n");
} else {
printf("ERROR: Out of range index should return NULL, returned offset: %ld\n", p-zl);
return 1;
}
printf("\n");
zfree(zl);
}
printf("Iterate list from 0 to end:\n");
{
zl = createList();
p = ziplistIndex(zl, 0);
while (ziplistGet(p, &entry, &elen, &value)) {
printf("Entry: ");
if (entry) {
if (elen && fwrite(entry,elen,1,stdout) == 0) perror("fwrite");
} else {
printf("%lld", value);
}
p = ziplistNext(zl,p);
printf("\n");
}
printf("\n");
zfree(zl);
}
printf("Iterate list from 1 to end:\n");
{
zl = createList();
p = ziplistIndex(zl, 1);
while (ziplistGet(p, &entry, &elen, &value)) {
printf("Entry: ");
if (entry) {
if (elen && fwrite(entry,elen,1,stdout) == 0) perror("fwrite");
} else {
printf("%lld", value);
}
p = ziplistNext(zl,p);
printf("\n");
}
printf("\n");
zfree(zl);
}
printf("Iterate list from 2 to end:\n");
{
zl = createList();
p = ziplistIndex(zl, 2);
while (ziplistGet(p, &entry, &elen, &value)) {
printf("Entry: ");
if (entry) {
if (elen && fwrite(entry,elen,1,stdout) == 0) perror("fwrite");
} else {
printf("%lld", value);
}
p = ziplistNext(zl,p);
printf("\n");
}
printf("\n");
zfree(zl);
}
printf("Iterate starting out of range:\n");
{
zl = createList();
p = ziplistIndex(zl, 4);
if (!ziplistGet(p, &entry, &elen, &value)) {
printf("No entry\n");
} else {
printf("ERROR\n");
}
printf("\n");
zfree(zl);
}
printf("Iterate from back to front:\n");
{
zl = createList();
p = ziplistIndex(zl, -1);
while (ziplistGet(p, &entry, &elen, &value)) {
printf("Entry: ");
if (entry) {
if (elen && fwrite(entry,elen,1,stdout) == 0) perror("fwrite");
} else {
printf("%lld", value);
}
p = ziplistPrev(zl,p);
printf("\n");
}
printf("\n");
zfree(zl);
}
printf("Iterate from back to front, deleting all items:\n");
{
zl = createList();
p = ziplistIndex(zl, -1);
while (ziplistGet(p, &entry, &elen, &value)) {
printf("Entry: ");
if (entry) {
if (elen && fwrite(entry,elen,1,stdout) == 0) perror("fwrite");
} else {
printf("%lld", value);
}
zl = ziplistDelete(zl,&p);
p = ziplistPrev(zl,p);
printf("\n");
}
printf("\n");
zfree(zl);
}
printf("Delete inclusive range 0,0:\n");
{
zl = createList();
zl = ziplistDeleteRange(zl, 0, 1);
ziplistRepr(zl);
zfree(zl);
}
printf("Delete inclusive range 0,1:\n");
{
zl = createList();
zl = ziplistDeleteRange(zl, 0, 2);
ziplistRepr(zl);
zfree(zl);
}
printf("Delete inclusive range 1,2:\n");
{
zl = createList();
zl = ziplistDeleteRange(zl, 1, 2);
ziplistRepr(zl);
zfree(zl);
}
printf("Delete with start index out of range:\n");
{
zl = createList();
zl = ziplistDeleteRange(zl, 5, 1);
ziplistRepr(zl);
zfree(zl);
}
printf("Delete with num overflow:\n");
{
zl = createList();
zl = ziplistDeleteRange(zl, 1, 5);
ziplistRepr(zl);
zfree(zl);
}
printf("Delete foo while iterating:\n");
{
zl = createList();
p = ziplistIndex(zl,0);
while (ziplistGet(p,&entry,&elen,&value)) {
if (entry && strncmp("foo",(char*)entry,elen) == 0) {
printf("Delete foo\n");
zl = ziplistDelete(zl,&p);
} else {
printf("Entry: ");
if (entry) {
if (elen && fwrite(entry,elen,1,stdout) == 0)
perror("fwrite");
} else {
printf("%lld",value);
}
p = ziplistNext(zl,p);
printf("\n");
}
}
printf("\n");
ziplistRepr(zl);
zfree(zl);
}
printf("Regression test for >255 byte strings:\n");
{
char v1[257] = {0}, v2[257] = {0};
memset(v1,'x',256);
memset(v2,'y',256);
zl = ziplistNew();
zl = ziplistPush(zl,(unsigned char*)v1,strlen(v1),ZIPLIST_TAIL);
zl = ziplistPush(zl,(unsigned char*)v2,strlen(v2),ZIPLIST_TAIL);
/* Pop values again and compare their value. */
p = ziplistIndex(zl,0);
assert(ziplistGet(p,&entry,&elen,&value));
assert(strncmp(v1,(char*)entry,elen) == 0);
p = ziplistIndex(zl,1);
assert(ziplistGet(p,&entry,&elen,&value));
assert(strncmp(v2,(char*)entry,elen) == 0);
printf("SUCCESS\n\n");
zfree(zl);
}
printf("Regression test deleting next to last entries:\n");
{
char v[3][257] = {{0}};
zlentry e[3] = {{.prevrawlensize = 0, .prevrawlen = 0, .lensize = 0,
.len = 0, .headersize = 0, .encoding = 0, .p = NULL}};
size_t i;
for (i = 0; i < (sizeof(v)/sizeof(v[0])); i++) {
memset(v[i], 'a' + i, sizeof(v[0]));
}
v[0][256] = '\0';
v[1][ 1] = '\0';
v[2][256] = '\0';
zl = ziplistNew();
for (i = 0; i < (sizeof(v)/sizeof(v[0])); i++) {
zl = ziplistPush(zl, (unsigned char *) v[i], strlen(v[i]), ZIPLIST_TAIL);
}
verify(zl, e);
assert(e[0].prevrawlensize == 1);
assert(e[1].prevrawlensize == 5);
assert(e[2].prevrawlensize == 1);
/* Deleting entry 1 will increase `prevrawlensize` for entry 2 */
unsigned char *p = e[1].p;
zl = ziplistDelete(zl, &p);
verify(zl, e);
assert(e[0].prevrawlensize == 1);
assert(e[1].prevrawlensize == 5);
printf("SUCCESS\n\n");
zfree(zl);
}
printf("Create long list and check indices:\n");
{
zl = ziplistNew();
char buf[32];
int i,len;
for (i = 0; i < 1000; i++) {
len = sprintf(buf,"%d",i);
zl = ziplistPush(zl,(unsigned char*)buf,len,ZIPLIST_TAIL);
}
for (i = 0; i < 1000; i++) {
p = ziplistIndex(zl,i);
assert(ziplistGet(p,NULL,NULL,&value));
assert(i == value);
p = ziplistIndex(zl,-i-1);
assert(ziplistGet(p,NULL,NULL,&value));
assert(999-i == value);
}
printf("SUCCESS\n\n");
zfree(zl);
}
printf("Compare strings with ziplist entries:\n");
{
zl = createList();
p = ziplistIndex(zl,0);
if (!ziplistCompare(p,(unsigned char*)"hello",5)) {
printf("ERROR: not \"hello\"\n");
return 1;
}
if (ziplistCompare(p,(unsigned char*)"hella",5)) {
printf("ERROR: \"hella\"\n");
return 1;
}
p = ziplistIndex(zl,3);
if (!ziplistCompare(p,(unsigned char*)"1024",4)) {
printf("ERROR: not \"1024\"\n");
return 1;
}
if (ziplistCompare(p,(unsigned char*)"1025",4)) {
printf("ERROR: \"1025\"\n");
return 1;
}
printf("SUCCESS\n\n");
zfree(zl);
}
printf("Merge test:\n");
{
/* create list gives us: [hello, foo, quux, 1024] */
zl = createList();
unsigned char *zl2 = createList();
unsigned char *zl3 = ziplistNew();
unsigned char *zl4 = ziplistNew();
if (ziplistMerge(&zl4, &zl4)) {
printf("ERROR: Allowed merging of one ziplist into itself.\n");
return 1;
}
/* Merge two empty ziplists, get empty result back. */
zl4 = ziplistMerge(&zl3, &zl4);
ziplistRepr(zl4);
if (ziplistLen(zl4)) {
printf("ERROR: Merging two empty ziplists created entries.\n");
return 1;
}
zfree(zl4);
zl2 = ziplistMerge(&zl, &zl2);
/* merge gives us: [hello, foo, quux, 1024, hello, foo, quux, 1024] */
ziplistRepr(zl2);
if (ziplistLen(zl2) != 8) {
printf("ERROR: Merged length not 8, but: %u\n", ziplistLen(zl2));
return 1;
}
p = ziplistIndex(zl2,0);
if (!ziplistCompare(p,(unsigned char*)"hello",5)) {
printf("ERROR: not \"hello\"\n");
return 1;
}
if (ziplistCompare(p,(unsigned char*)"hella",5)) {
printf("ERROR: \"hella\"\n");
return 1;
}
p = ziplistIndex(zl2,3);
if (!ziplistCompare(p,(unsigned char*)"1024",4)) {
printf("ERROR: not \"1024\"\n");
return 1;
}
if (ziplistCompare(p,(unsigned char*)"1025",4)) {
printf("ERROR: \"1025\"\n");
return 1;
}
p = ziplistIndex(zl2,4);
if (!ziplistCompare(p,(unsigned char*)"hello",5)) {
printf("ERROR: not \"hello\"\n");
return 1;
}
if (ziplistCompare(p,(unsigned char*)"hella",5)) {
printf("ERROR: \"hella\"\n");
return 1;
}
p = ziplistIndex(zl2,7);
if (!ziplistCompare(p,(unsigned char*)"1024",4)) {
printf("ERROR: not \"1024\"\n");
return 1;
}
if (ziplistCompare(p,(unsigned char*)"1025",4)) {
printf("ERROR: \"1025\"\n");
return 1;
}
printf("SUCCESS\n\n");
zfree(zl);
}
printf("Stress with random payloads of different encoding:\n");
{
int i,j,len,where;
unsigned char *p;
char buf[1024];
int buflen;
list *ref;
listNode *refnode;
/* Hold temp vars from ziplist */
unsigned char *sstr;
unsigned int slen;
long long sval;
for (i = 0; i < 20000; i++) {
zl = ziplistNew();
ref = listCreate();
listSetFreeMethod(ref,(void (*)(void*))sdsfree);
len = rand() % 256;
/* Create lists */
for (j = 0; j < len; j++) {
where = (rand() & 1) ? ZIPLIST_HEAD : ZIPLIST_TAIL;
if (rand() % 2) {
buflen = randstring(buf,1,sizeof(buf)-1);
} else {
switch(rand() % 3) {
case 0:
buflen = sprintf(buf,"%lld",(0LL + rand()) >> 20);
break;
case 1:
buflen = sprintf(buf,"%lld",(0LL + rand()));
break;
case 2:
buflen = sprintf(buf,"%lld",(0LL + rand()) << 20);
break;
default:
assert(NULL);
}
}
/* Add to ziplist */
zl = ziplistPush(zl, (unsigned char*)buf, buflen, where);
/* Add to reference list */
if (where == ZIPLIST_HEAD) {
listAddNodeHead(ref,sdsnewlen(buf, buflen));
} else if (where == ZIPLIST_TAIL) {
listAddNodeTail(ref,sdsnewlen(buf, buflen));
} else {
assert(NULL);
}
}
assert(listLength(ref) == ziplistLen(zl));
for (j = 0; j < len; j++) {
/* Naive way to get elements, but similar to the stresser
* executed from the Tcl test suite. */
p = ziplistIndex(zl,j);
refnode = listIndex(ref,j);
assert(ziplistGet(p,&sstr,&slen,&sval));
if (sstr == NULL) {
buflen = sprintf(buf,"%lld",sval);
} else {
buflen = slen;
memcpy(buf,sstr,buflen);
buf[buflen] = '\0';
}
assert(memcmp(buf,listNodeValue(refnode),buflen) == 0);
}
zfree(zl);
listRelease(ref);
}
printf("SUCCESS\n\n");
}
printf("Stress with variable ziplist size:\n");
{
stress(ZIPLIST_HEAD,100000,16384,256);
stress(ZIPLIST_TAIL,100000,16384,256);
}
return 0;
}
#endif

3.参考文献

redis_reading