BlobNode 配置

BlobNode 是单机存储引擎模块,主要负责组织数据落盘、磁盘读取数据、磁盘删除数据以及后台任务的执行等。

BlobNode的配置是基于公有配置,以下配置说明主要针对于BlobNode的私有配置。

配置说明

关键配置

配置项说明必需
公有配置如服务端口、运行日志以及审计日志等,参考基础服务配置章节
disks要注册的磁盘路径列表
disable_sync是否关闭磁盘sync,值为true表示关闭sync,可以提高写性能
rack所在机架编号,clustermgr打开机架隔离时需要此字段
host本机的blobnode服务地址,需要注册到clustermgr
must_mount_point校验注册路径是否是挂载路径,生产环境建议打开
data_qos数据qos分层限流,生产环境建议打开
meta_config元数据相关配置,包含rocksdb的cache大小等
clustermgrclustermgr的服务地址信息等

全部配置

{
	"cluster_id": "集群id",
	"idc": "机房",
	"rack": "机架",
	"host": "服务地址ip:端口",
	"dropped_bid_record": {
		"dir": "下线的bid记录目录",
		"chunkbits": "n: 单个文件大小,2的n次方字节"
	},
	"disks": [
		{
			"auto_format": "是否自动创建目录",
			"disable_sync": "是否关闭磁盘sync",
			"path": "数据存放目录",
			"max_chunks": "单盘最大的chunk数量限制"
		},
		{
			"auto_format": "同上",
			"disable_sync": "同上",
			"path": "同上:每个磁盘都需要配置,会注册到clustermgr",
			"max_chunks": "同上"
		}
	],
	"disk_config": {
		"disk_reserved_space_B": "单盘保留大小,可写空间会减去该值,默认60GB",
		"compact_reserved_space_B": "留给压缩的单盘保留大小,默认20GB",
		"chunk_protection_M": "release chunk的保护周期,最后修改时间到此刻没有超过保护周期则不允许release",
		"chunk_compact_interval_S": "compact的定时任务周期",
		"chunk_clean_interval_S": "chunk清理的定时任务周期",
		"chunk_gc_create_time_protection_M": "清理chunk时判断创建时间的保护周期",
		"chunk_gc_modify_time_protection_M": "清理chunk时判断修改时间的保护周期",
		"disk_usage_interval_S": "更新磁盘空间使用情况的定时任务周期",
		"disk_clean_trash_interval_S": "磁盘清理垃圾数据的定时任务周期",
		"disk_trash_protection_M": "磁盘垃圾数据的保护周期",
		"allow_clean_trash": "是否允许清理垃圾",
		"disable_modify_in_compacting": "压缩中是否不允许修改chunk",
		"compact_min_size_threshold": "chunk大小满足压缩条件的最小值",
		"compact_trigger_threshold": "chunk size达到该值触发压缩",
		"compact_empty_rate_threshold": "chunk空洞率达到该值允许压缩",
		"need_compact_check": "压缩完成后,是否巡检压缩前后的blob,确保压缩前后数据一致",
		"allow_force_compact": "是否允许接口强制进行压缩,跳过压缩条件",
		"compact_batch_size": "执行压缩时每一批次的bid数量",
		"must_mount_point": "数据存放目录是否强制是挂载点",
		"metric_report_interval_S": "metric上报的定时任务周期",
		"data_qos": {
			"disk_bandwidth_MBPS": "单盘整体带宽阈值,带宽达到该值会调整每个level的限流带宽为bandwidth_MBPS*factor",
			"disk_iops": "单盘整体iops阈值,iops达到该值会调整每个level的限流iops为level配置的iops*factor",
			"flow_conf": {
				"level0": {
					"bandwidth_MBPS": "(level0是用户读写io)限流带宽MB/s",
					"iops": "限流IOPS",
					"factor": "限流因子"
				},
				"level1": {
					"bandwidth_MBPS": "(level1是shard repair io)限流带宽MB/s",
					"iops": "同上",
					"factor": "同上"
				},
				"level2": {
					"bandwidth_MBPS": "(level2是disk repair, delete, compact io)限流带宽MB/s",
					"iops": "同上",
					"factor": "同上"
				},
				"level3": {
					"bandwidth_MBPS": "(level3是balance, drop, manual migrate io)限流带宽MB/s",
					"iops": "同上",
					"factor": "同上"
				},
				"level4": {
					"bandwidth_MBPS": "(level4是inspect io)限流带宽MB/s",
					"iops": "同上",
					"factor": "同上"
				}
			}
		}
	},
	"meta_config": {
		"meta_root_prefix": "配置统一的meta数据存放目录,可以配置到ssd盘提高元数据读写速度,默认不配置",
		"batch_process_count": "元数据批量处理请求的个数,包含删除和写",
		"support_inline": "是否开启小文件内联写到元数据存rocksdb",
		"tinyfile_threshold_B": "小文件阈值,可以配置小于等于128k",
		"write_pri_ratio": "元数据批量处理写请求的比例,具体个数为batch_process_count*write_pri_ratio",
		"sync": "是否开启磁盘sync",
		"rocksdb_option": {
			"lrucache": "缓存大小",
			"write_buffer_size": "rocksdb写buffer大小"
		},
		"meta_qos": {
			"level0": {
				"iops": "level0是用户读写io的iops限制,元数据限流配置"
			},
			"level1": {
				"iops": "level1是shard repair io的iops限制"
			},
			"level2": {
				"iops": "level2是disk repair, delete, compact io的iops限制"
			},
			"level3": {
				"iops": "level3是balance, drop, manual migrate io的iops限制"
			},
			"level4": {
				"iops": "level4是inspect io的iops限制"
			}
		}
	},
	"clustermgr": {
		"hosts": "clustermgr服务地址"
	},
	"blobnode": {
		"client_timeout_ms": "后台任务用到的blobnode client的超时时间"
	},
	"scheduler": {
		"host_sync_interval_ms": "后台任务用到的scheduler client的后端节点同步时间"
	},
	"chunk_protection_period_S": "过期epoch chunk判断创建时间的保护周期",
	"put_qps_limit_per_disk": "单盘写并发数控制",
	"get_qps_limit_per_disk": "单盘读并发数控制",
	"get_qps_limit_per_key": "单个shard的读并发数控制",
	"delete_qps_limit_per_disk": "单盘删除的并发数控制",
	"shard_repair_concurrency": "后台任务shard repair的并发数控制",
	"flock_filename": "进程文件锁路径"
}

示例配置

{    
    "bind_addr": ":8889",    
    "log": {		
        "level": 2    
    },    
    "cluster_id": 10001,    
    "idc": "bjht",    
    "rack": "HT02-B11-F4-402-0406",    
    "host": "http://10.39.34.185:8889",                                             
    "dropped_bid_record": {       
        "dir": "/home/service/ebs-blobnode/_package/dropped_bids/",       
        "chunkbits": 29     
    },    
    "disks": [      
        {"auto_format": true,"disable_sync": true,"path": "/home/service/var/data1"},
        {"auto_format": true,"disable_sync": true,"path": "/home/service/var/data2"}
    ],    
    "disk_config": {        
        "chunk_clean_interval_S": 60,        
        "chunk_protection_M": 30,        
        "disk_clean_trash_interval_S": 60,        
        "disk_trash_protection_M": 1440,        
        "metric_report_interval_S": 300,        
        "need_compact_check": true,        
        "allow_force_compact": true,        
        "allow_clean_trash": true,        
        "must_mount_point": true,        
        "data_qos": {            
            "disk_bandwidth_MBPS": 200,            
            "disk_iops": 8000,            
            "flow_conf": {                                                                                             
                "level0": { 					
                    "bandwidth_MBPS": 200, 
                    "iops": 4000, 
                    "factor": 1
                },                
                "level1": {							
                    "bandwidth_MBPS": 40, 
                    "iops": 2000, 
                    "factor": 0.5
                },                
                "level2": {							
                    "bandwidth_MBPS": 40, 
                    "iops": 2000, 
                    "factor": 0.5
                },                
                "level3": {							
                    "bandwidth_MBPS": 40, 
                    "iops": 2000, 
                    "factor": 0.5
                },                
                "level4": {							
                    "bandwidth_MBPS": 20, 
                    "iops": 1000, 
                    "factor": 0.5
                }            
            }        
        }    
    },    
    "meta_config": {        
        "sync": false,        
        "rocksdb_option": {            
            "lrucache": 268435456,            
            "write_buffer_size": 52428800        
        },        
        "meta_qos": {             
            "level0": { 
                "iops": 8000 
            },             
            "level1": { 
                "iops": 8000 
            },             
            "level2": { 
                "iops": 8000 
            },             
            "level3": { 
                "iops": 8000 
            },             
            "level4": { 
                "iops": 8000 
            }        
        }    
    },    
    "clustermgr": {        
        "hosts": [      
            "http://10.39.30.78:9998",      
            "http://10.39.32.224:9998",      
            "http://10.39.32.234:9998"
        ],        
        "transport_config": {         	
            "max_conns_per_host": 4,            
            "auth": {                
                "enable_auth": false,                
                "secret": "b2e5e2ed-6fca-47ce-bfbc-5e8f0650603b"            
            }        
        }    
    },    
    "blobnode": {    	
        "client_timeout_ms": 5000    
    },    
    "scheduler": {    	
        "host_sync_interval_ms": 3600000    
    },    
    "chunk_protection_period_S": 600,     
    "put_qps_limit_per_disk": 1024,    
    "get_qps_limit_per_disk": 1024,    
    "get_qps_limit_per_key": 1024,    
    "delete_qps_limit_per_disk": 64,      
    "shard_repair_concurrency": 100,    
    "flock_filename": "/home/service/ebs-blobnode/_package/run/blobnode.0.flock",    
    "auditlog": {        
        "logdir": "/home/service/ebs-blobnode/_package/run/auditlog/ebs-blobnode",        
        "chunkbits": 29,        
        "log_file_suffix": ".log",        
        "backup": 10,        
        "filters": [
            {"should": {"match": {"path": ["list", "metrics", "/shard/get/"]}}}
        ],
        "metric_config": {            
            "idc": "bjht",            
            "service": "BLOBNODE",            
            "team": "ocs",            
            "enable_http_method": true,            
            "enable_req_length_cnt": true,            
            "enable_resp_length_cnt": true,            
            "enable_resp_duration": true,            
            "max_api_level": 3,            
            "size_buckets": [
                0, 
                65536, 
                131072, 
                262144, 
                524288, 
                1048576, 
                4194304, 
                8388608, 
                16777216, 
                33554432
            ]        
        }    
    }
}
在github上编辑