parse and decode Redis AOF file with Python
decode and parse
Redis
aof logappendonly.aof
into plain text, and upload it toHDFS
or load it intoHive
for further process.
RESP
Redis采用RESP
协议 (REdis Serialization Protocol),AOF
文件则是对该协议的持久化记录,一条命令对应的RESP
格式如下:
*<参数数量>\r\n
$<第一个参数/命令的字节数量>\r\n
<参数1>\r\n
...
$<参数N的字节数量>\r\n
<参数N>\r\n
比如,set key:000000000007 VXK
RESP
格式的aof
文件是
*3
$3
SET
$16
key:000000000007
$3
VXK
parse AOF
#!/usr/bin/env python3
import argparse
def decode_aof(filename):
key_no = 0
v_sign = False
print_sign = False
key = ''
val = ''
with open(filename, 'r') as f:
for line in f:
if line.startswith('*'):
key_no = 0
v_sign = False
if print_sign == 1:
print(key, '\t', val)
print_sign = 0
val = ''
if key_no == 4:
key = line.replace('\n', '')
key_no += 1
v_sign = True
continue
if v_sign is True and not line.startswith('$'):
val += str(line.strip('\r\n'))
print_sign = True
key_no += 1
print(key, '\t', val)
if __name__ == '__main__':
p = argparse.ArgumentParser(description='decode tool', conflict_handler='error')
p.add_argument('aoffile', type=str, nargs='*', help='aoffile')
args = p.parse_args()
if not args.aoffile:
p.parse_args(['-h'])
else:
decode_aof(args.aoffile[0])
test example, appendonly.aof
$6
SELECT
$1
0
*3
$3
SET
$16
key:000000000007
$3
VXK
*3
$3
SET
$16
key:000000000000
$3
VXK
*3
$3
SET
$16
key:000000000004
$3
VXK
*3
$3
SET
$16
key:000000000005
$3
VXK
*3
$3
SET
$16
key:000000000006
$3
VXK
*3
$3
SET
$16
key:000000000004
$3
VXK
*3
$3
SET
$16
key:000000000008
$3
VXK
*3
$3
SET
$16
key:000000000005
$3
VXK
*3
$3
SET
$16
key:000000000005
$3
VXK
*3
$3
SET
$16
key:000000000006
$3
VXK
*3
$4
SADD
$5
myset
$20
element:000000000006
*3
$4
SADD
$5
myset
$20
element:000000000008
*3
$4
SADD
$5
myset
$20
element:000000000007
*3
$4
SADD
$5
myset
$20
element:000000000009
*3
$4
SADD
$5
myset
$20
element:000000000003
*3
$4
SADD
$5
myset
$20
element:000000000001
*4
$4
HSET
$6
myhash
$20
element:000000000004
$3
VXK
*4
$4
HSET
$6
myhash
$20
element:000000000009
$3
VXK
*4
$4
HSET
$6
myhash
$20
element:000000000007
$3
VXK
*4
$4
HSET
$6
myhash
$20
element:000000000001
$3
VXK
*4
$4
HSET
$6
myhash
$20
element:000000000002
$3
VXK
*4
$4
HSET
$6
myhash
$20
element:000000000001
$3
VXK
*4
$4
HSET
$6
myhash
$20
element:000000000005
$3
VXK
*4
$4
HSET
$6
myhash
$20
element:000000000008
$3
VXK
*4
$4
HSET
$6
myhash
$20
element:000000000009
$3
VXK
*4
$4
HSET
$6
myhash
$20
element:000000000004
$3
VXK
*4
$4
ZADD
$6
myzset
$12
000000000006
$20
element:000000000008
*4
$4
ZADD
$6
myzset
$12
000000000006
$20
element:000000000002
*4
$4
ZADD
$6
myzset
$12
000000000006
$20
element:000000000004
*4
$4
ZADD
$6
myzset
$12
000000000007
$20
element:000000000004
*4
$4
ZADD
$6
myzset
$12
000000000009
$20
element:000000000004
*4
$4
ZADD
$6
myzset
$12
000000000000
$20
element:000000000006
*4
$4
ZADD
$6
myzset
$12
000000000005
$20
element:000000000009
*4
$4
ZADD
$6
myzset
$12
000000000005
$20
element:000000000006
*4
$4
ZADD
$6
myzset
$12
000000000002
$20
element:000000000006
*4
$4
ZADD
$6
myzset
$12
000000000003
$20
element:000000000004
The result and output
key:000000000007 VXK
key:000000000000 VXK
key:000000000004 VXK
key:000000000005 VXK
key:000000000006 VXK
key:000000000004 VXK
key:000000000008 VXK
key:000000000005 VXK
key:000000000005 VXK
key:000000000006 VXK
myset element:000000000006
myset element:000000000008
myset element:000000000007
myset element:000000000009
myset element:000000000003
myset element:000000000001
myhash element:000000000004VXK
myhash element:000000000009VXK
myhash element:000000000007VXK
myhash element:000000000001VXK
myhash element:000000000002VXK
myhash element:000000000001VXK
myhash element:000000000005VXK
myhash element:000000000008VXK
myhash element:000000000009VXK
myhash element:000000000004VXK
myzset 000000000006element:000000000008
myzset 000000000006element:000000000002
myzset 000000000006element:000000000004
myzset 000000000007element:000000000004
myzset 000000000009element:000000000004
myzset 000000000000element:000000000006
myzset 000000000005element:000000000009
myzset 000000000005element:000000000006
myzset 000000000002element:000000000006
myzset 000000000003element:000000000004
Disclaimer
- License under
CC BY-NC 4.0
- Copyright issue feedback
me#imzye.me
, replace # with @ - Not all the commands and scripts are tested in production environment, use at your own risk
- No privacy information is collected here