parse and decode Redis AOF file with Python

decode and parse Redis aof log appendonly.aof into plain text, and upload it to HDFS or load it into Hive for further process.

RESP

Redis采用RESP协议 (REdis Serialization Protocol)，AOF文件则是对该协议的持久化记录，一条命令对应的RESP格式如下：

*<参数数量>\r\n
$<第一个参数/命令的字节数量>\r\n
<参数1>\r\n
...
$<参数N的字节数量>\r\n
<参数N>\r\n

比如，set key:000000000007 VXK RESP格式的aof文件是

*3
$3
SET
$16
key:000000000007
$3
VXK

parse AOF

#!/usr/bin/env python3
import argparse

def decode_aof(filename):
    key_no = 0
    v_sign = False
    print_sign = False
    key = ''
    val = ''
    with open(filename, 'r') as f:
        for line in f:
            if line.startswith('*'):
                key_no = 0
                v_sign = False
                if print_sign == 1:
                    print(key, '\t', val)
                    print_sign = 0
                    val = ''
            if key_no == 4:
                key = line.replace('\n', '')
                key_no += 1
                v_sign = True
                continue
            if v_sign is True and not line.startswith('$'):
                val += str(line.strip('\r\n'))
                print_sign = True
            key_no += 1
    print(key, '\t', val)


if __name__ == '__main__':
    p = argparse.ArgumentParser(description='decode tool', conflict_handler='error')
    p.add_argument('aoffile', type=str, nargs='*', help='aoffile')
    args = p.parse_args()
    if not args.aoffile:
        p.parse_args(['-h'])
    else:
        decode_aof(args.aoffile[0])

test example, appendonly.aof

$6
SELECT
$1
0
*3
$3
SET
$16
key:000000000007
$3
VXK
*3
$3
SET
$16
key:000000000000
$3
VXK
*3
$3
SET
$16
key:000000000004
$3
VXK
*3
$3
SET
$16
key:000000000005
$3
VXK
*3
$3
SET
$16
key:000000000006
$3
VXK
*3
$3
SET
$16
key:000000000004
$3
VXK
*3
$3
SET
$16
key:000000000008
$3
VXK
*3
$3
SET
$16
key:000000000005
$3
VXK
*3
$3
SET
$16
key:000000000005
$3
VXK
*3
$3
SET
$16
key:000000000006
$3
VXK
*3
$4
SADD
$5
myset
$20
element:000000000006
*3
$4
SADD
$5
myset
$20
element:000000000008
*3
$4
SADD
$5
myset
$20
element:000000000007
*3
$4
SADD
$5
myset
$20
element:000000000009
*3
$4
SADD
$5
myset
$20
element:000000000003
*3
$4
SADD
$5
myset
$20
element:000000000001
*4
$4
HSET
$6
myhash
$20
element:000000000004
$3
VXK
*4
$4
HSET
$6
myhash
$20
element:000000000009
$3
VXK
*4
$4
HSET
$6
myhash
$20
element:000000000007
$3
VXK
*4
$4
HSET
$6
myhash
$20
element:000000000001
$3
VXK
*4
$4
HSET
$6
myhash
$20
element:000000000002
$3
VXK
*4
$4
HSET
$6
myhash
$20
element:000000000001
$3
VXK
*4
$4
HSET
$6
myhash
$20
element:000000000005
$3
VXK
*4
$4
HSET
$6
myhash
$20
element:000000000008
$3
VXK
*4
$4
HSET
$6
myhash
$20
element:000000000009
$3
VXK
*4
$4
HSET
$6
myhash
$20
element:000000000004
$3
VXK
*4
$4
ZADD
$6
myzset
$12
000000000006
$20
element:000000000008
*4
$4
ZADD
$6
myzset
$12
000000000006
$20
element:000000000002
*4
$4
ZADD
$6
myzset
$12
000000000006
$20
element:000000000004
*4
$4
ZADD
$6
myzset
$12
000000000007
$20
element:000000000004
*4
$4
ZADD
$6
myzset
$12
000000000009
$20
element:000000000004
*4
$4
ZADD
$6
myzset
$12
000000000000
$20
element:000000000006
*4
$4
ZADD
$6
myzset
$12
000000000005
$20
element:000000000009
*4
$4
ZADD
$6
myzset
$12
000000000005
$20
element:000000000006
*4
$4
ZADD
$6
myzset
$12
000000000002
$20
element:000000000006
*4
$4
ZADD
$6
myzset
$12
000000000003
$20
element:000000000004

The result and output

key:000000000007   VXK
key:000000000000   VXK
key:000000000004   VXK
key:000000000005   VXK
key:000000000006   VXK
key:000000000004   VXK
key:000000000008   VXK
key:000000000005   VXK
key:000000000005   VXK
key:000000000006   VXK
myset   element:000000000006
myset   element:000000000008
myset   element:000000000007
myset   element:000000000009
myset   element:000000000003
myset   element:000000000001
myhash   element:000000000004VXK
myhash   element:000000000009VXK
myhash   element:000000000007VXK
myhash   element:000000000001VXK
myhash   element:000000000002VXK
myhash   element:000000000001VXK
myhash   element:000000000005VXK
myhash   element:000000000008VXK
myhash   element:000000000009VXK
myhash   element:000000000004VXK
myzset   000000000006element:000000000008
myzset   000000000006element:000000000002
myzset   000000000006element:000000000004
myzset   000000000007element:000000000004
myzset   000000000009element:000000000004
myzset   000000000000element:000000000006
myzset   000000000005element:000000000009
myzset   000000000005element:000000000006
myzset   000000000002element:000000000006
myzset   000000000003element:000000000004

Disclaimer

License under CC BY-NC 4.0
Copyright issue feedback me#imzye.me, replace # with @
Not all the commands and scripts are tested in production environment, use at your own risk
No privacy information is collected here

Try iOS App

Yi's Blog