initial Module
初始化流程
初始化Suricata instance 用来保存程序当前的一些状态、标志等上下文环境，通常是用来作为参数传递给各个模块的子函数memset(suri, 0x00, sizeof(*suri));// pointer to argv[0] suri->progname = progname;//运行模式suri->run_mode = RUNMODE_UNKNOWN;memset(suri->pcap_dev, 0, sizeof(suri->pcap_dev));//签名文件suri->sig_file = NULL;suri->sig_file_exclusive = FALSE;suri->pid_filename = NULL;//正则参数suri->regex_arg = NULL;//延迟检测开关suri->delayed_detect = 0;//守护进程suri->daemon = 0;//离线模式suri->offline = 0;//详细模式suri->verbose = 0;/* use -1 as unknown */suri->checksum_validation = -1;//是否检测g_detect_disabled = suri->disabled_detect = 1;
初始化原子变量engine_stage –> 记录程序当前的运行阶段：SURICATA_INIT、SURICATA_RUNTIME、SURICATA_FINALIZE
suricata_context初始化//打印相关 console or file 以及 level suricata_context.SCLogMessage = SCLogMessage;// 文件io相关的函数suricata_context.FileOpenFileWithId = FileOpenFileWithId;suricata_context.FileCloseFileById = FileCloseFileById;suricata_context.FileAppendDataById = FileAppendDataById;suricata_context.FileAppendGAPById = FileAppendGAPById;suricata_context.FileContainerRecycle = FileContainerRecycle;suricata_context.FilePrune = FilePrune;suricata_context.FileSetTx = FileContainerSetTx;
以环境变量或缺省值对日志模块初始化
op_iface日志输出接口：console or file or syslog文件指针文件锁日志格式和levelop_filter_regex 只输出匹配到这个正则的文件名
设置当前主线程名字为“Suricata-Main”。在gdb调试时info threads可以看到各个线程名
初始化ParserSize模块，正则预编译，用于解析类似“10Mb”这种大小参数
#define PARSE_REGEX "^\\s*(\\d+(?:.\\d+)?)\\s*([a-zA-Z]{2})?\\s*$"
注册各种运行模式。Suricata对“运行模式”这个概念也进行了封装。运行模式存储在runmodes数组中，定义为RunModes runmodes[RUNMODE_USER_MAX]
enum RunModes {RUNMODE_UNKNOWN = 0,RUNMODE_PCAP_DEV,RUNMODE_PCAP_FILE,RUNMODE_PFRING,RUNMODE_NFQ,RUNMODE_NFLOG,RUNMODE_IPFW,RUNMODE_ERF_FILE,RUNMODE_DAG,RUNMODE_AFP_DEV,RUNMODE_NETMAP,RUNMODE_UNITTEST,RUNMODE_NAPATECH,RUNMODE_UNIX_SOCKET,RUNMODE_WINDIVERT,RUNMODE_PLUGIN,}typedef struct RunModes_ {int cnt;RunMode *runmodes;} RunModes;
注册各种运行模式 以IDS Pcap运行模式为例，它由3个子运行模式组成，底层由realloc分配空间，IDS Pcap Mode =RunModeIdsPcapSingle（single thread pcap processing）+ RunModeIdsPcapAutoFp（默认子运行模式）+RunModeIdsPcapWorkers（Workers version of the PCAP LIVE processing）
等后续初始化阶段确定了具体的运行模式后，就会调用这3个注册的对应的初始化函数，对该模式下的运行环境进行进一步配置
/*当前子模式：RunMode *mode = &runmodes[runmode].runmodes[runmodes[runmode].cnt];*/RunModeIdsPcapRegister();RunModeFilePcapRegister();RunModeIdsPfringRegister();RunModeIpsNFQRegister();RunModeIpsIPFWRegister();RunModeErfFileRegister();RunModeErfDagRegister();RunModeNapatechRegister();RunModeIdsAFPRegister();RunModeIdsNetmapRegister();RunModeIdsNflogRegister();RunModeUnixSocketRegister();RunModeIpsWinDivertRegister();
初始化配置模块，为后续解析Suricata.yaml建立配置节点树的根节点root，底层调用为alloc
typedef struct ConfNode_ {//filename: fast.log name:valchar *name;char *val;/**< Flag that sets this nodes value as final. */int final;//父节点struct ConfNode_ *parent;//头节点，next节点TAILQ_HEAD(, ConfNode_) head;TAILQ_ENTRY(ConfNode_) next;} ConfNode;
9. 命令行参数解析
//监控设备注册typedef struct LiveDeviceName_ {char *dev; /**< the device (e.g. "eth0") */TAILQ_ENTRY(LiveDeviceName_) next;} LiveDeviceName;pd = SCCalloc(1, sizeof(LiveDeviceName));TAILQ_INSERT_TAIL(&pre_live_devices, pd, next);g_engine_mode默认为IDS，IPS是可以drop包(--simulate-ips)EngineModeSetIPS();短命令行参数解析，并添加到Conf Tree中：ConfGetNodeOrCreate + setVal“-v”选项可多次使用，每个v都能将当前日志等级提升一级suri->verbose++;设置全局的run_mode变量，校验daemon模式是否兼容run_mode（Pcap文件模式及单元测试模式都不能在daemon开启下进行）10. GlobalsInitPreConfig获取当前时间所用的spin lock，以及设置时区（调用tzset()即可） 多模快速匹配链表维护，按优先级排序 sm_fp_support_smlist_list，底层SCMalloc 匹配阈值关键字相关，正则预编译
LoadYamlConfig
调用LoadYamlConfig读取Yaml格式配置文件。Yaml格式解析是通过libyaml库来完成的，解析的结果存储在配置节点树（见conf.c）中
SCLogLoadConfig
根据之前转化好的配置树，初始化日志模块一些全局配置
SCLogDebug("sc_log_global_log_level: %d", sc_log_global_log_level);SCLogDebug("sc_lc->log_format: %s", sc_log_config->log_format); SCLogDebug("SCLogSetOPFilter: filter: %s", sc_log_config->op_filter);typedef struct SCLogConfig_ {char *startup_message; SCLogLevel log_level; char *log_format;char *op_filter; /* compiled pcre filter expression */pcre *op_filter_regex;pcre_extra *op_filter_regex_study;/* op ifaces used */ SCLogOPIfaceCtx *op_ifaces;/* no of op ifaces */ uint8_t op_ifaces_cnt; } SCLogConfig;
PostConfLoadedSetup配置生效后，模块setup
MpmTableSetup：设置多模式匹配表，该表中每一项就是一个实现了某种多模式匹配算法（如WuManber、AC）的匹配器 ac,ac-ks,ac-bs,hs
SpmTableSetup:设置单模匹配表， bm,hs
typedef struct MpmTableElmt_ {const char *name;void (*InitCtx)(struct MpmCtx_ *);void (*InitThreadCtx)(struct MpmCtx_ *, struct MpmThreadCtx_ *);void (*DestroyCtx)(struct MpmCtx_ *);void (*DestroyThreadCtx)(struct MpmCtx_ *, struct MpmThreadCtx_ *);/** function pointers for adding patterns to the mpm ctx.** \param mpm_ctx Mpm context to add the pattern to* \param pattern pointer to the pattern* \param pattern_len length of the pattern in bytes* \param offset pattern offset setting* \param depth pattern depth setting* \param pid pattern id* \param sid signature _internal_ id* \param flags pattern flags*/int (*AddPattern)(struct MpmCtx_ *, uint8_t *, uint16_t, uint16_t, uint16_t, uint32_t, SigIntId, uint8_t);int (*AddPatternNocase)(struct MpmCtx_ *, uint8_t *, uint16_t, uint16_t, uint16_t, uint32_t, SigIntId, uint8_t);int (*Prepare)(struct MpmCtx_ *);uint32_t (*Search)(const struct MpmCtx_ *, struct MpmThreadCtx_ *, PrefilterRuleStore *, const uint8_t *, uint32_t);void (*PrintCtx)(struct MpmCtx_ *);void (*PrintThreadCtx)(struct MpmThreadCtx_ *);void (*RegisterUnittests)(void);uint8_t flags;} MpmTableElmt;extern MpmTableElmt mpm_table[MPM_TABLE_SIZE];typedef struct MpmCtx_ {void *ctx;uint8_t mpm_type;uint8_t flags;uint16_t maxdepth;/* unique patterns */uint32_t pattern_cnt;uint16_t minlen;uint16_t maxlen;uint32_t memory_cnt;uint32_t memory_size;uint32_t max_pat_id;/* hash used during ctx initialization */MpmPattern **init_hash;} MpmCtx;typedef struct SCACCtx_ {/* pattern arrays. We need this only during the goto table creation phase */MpmPattern **parray;/* no of states used by ac */uint32_t state_count;uint32_t pattern_id_bitarray_size;/* the all important memory hungry state_table */SC_AC_STATE_TYPE_U16 (*state_table_u16)[256];/* the all important memory hungry state_table */SC_AC_STATE_TYPE_U32 (*state_table_u32)[256];/* goto_table, failure table and output table. Needed to create state_table.* Will be freed, once we have created the state_table */int32_t (*goto_table)[256];int32_t *failure_table;SCACOutputTable *output_table;SCACPatternList *pid_pat_list;/* the size of each state */uint32_t single_state_size;uint32_t allocated_state_count;} SCACCtx;
存储相关初始化 如tag关键字，由于这类信息依赖于规则，同时出现的频率不高,申请内存
STORAGE_HOST,STORAGE_FLOW,STORAGE_IPPAIR,STORAGE_DEVICE memset(&storage_max_id, 0x00, sizeof(storage_max_id)); storage_list = NULL; storage_map = NULL; storage_registraton_closed = 0; 注册RegisterFlowBypassInfo旁路流量卸载STORAGE_FLOW
协议解析器注册
AppLayerParserProtoCtx ctxs[FLOW_PROTO_MAX][ALPROTO_MAX];enum {FLOW_PROTO_TCP = 0,FLOW_PROTO_UDP,FLOW_PROTO_ICMP,FLOW_PROTO_DEFAULT,/* should be last */FLOW_PROTO_MAX,};enum AppProtoEnum {ALPROTO_UNKNOWN = 0,ALPROTO_HTTP,ALPROTO_FTP,ALPROTO_SMTP,ALPROTO_TLS, /* SSLv2, SSLv3 & TLSv1 */ALPROTO_SSH,ALPROTO_IMAP,ALPROTO_JABBER,ALPROTO_SMB,ALPROTO_DCERPC,ALPROTO_IRC,ALPROTO_DNS,ALPROTO_MODBUS,ALPROTO_ENIP,ALPROTO_DNP3,ALPROTO_NFS,ALPROTO_NTP,ALPROTO_FTPDATA,ALPROTO_TFTP,ALPROTO_IKEV2,ALPROTO_KRB5,ALPROTO_DHCP,ALPROTO_SNMP,ALPROTO_SIP,ALPROTO_RFB,ALPROTO_MQTT,ALPROTO_TEMPLATE,ALPROTO_TEMPLATE_RUST,ALPROTO_RDP,ALPROTO_HTTP2,/* used by the probing parser when alproto detection fails* permanently for that particular stream */ALPROTO_FAILED,#ifdef UNITTESTSALPROTO_TEST,#endif /* UNITESTS *//* keep last */ALPROTO_MAX,};/*** \brief App layer protocol parser context.*/typedef struct AppLayerParserProtoCtx_{/* 0 - to_server, 1 - to_client. */AppLayerParserFPtr Parser[2];bool logger;uint32_t logger_bits; /**< registered loggers for this proto */void *(*StateAlloc)(void *, AppProto);void (*StateFree)(void *);void (*StateTransactionFree)(void *, uint64_t);void *(*LocalStorageAlloc)(void);void (*LocalStorageFree)(void *);void (*Truncate)(void *, uint8_t);FileContainer *(*StateGetFiles)(void *, uint8_t);AppLayerDecoderEvents *(*StateGetEvents)(void *);int (*StateGetProgress)(void *alstate, uint8_t direction);uint64_t (*StateGetTxCnt)(void *alstate);void *(*StateGetTx)(void *alstate, uint64_t tx_id);AppLayerGetTxIteratorFunc StateGetTxIterator;int (*StateGetProgressCompletionStatus)(uint8_t direction);int (*StateGetEventInfoById)(int event_id, const char **event_name,AppLayerEventType *event_type);int (*StateGetEventInfo)(const char *event_name,int *event_id, AppLayerEventType *event_type);DetectEngineState *(*GetTxDetectState)(void *tx);int (*SetTxDetectState)(void *tx, DetectEngineState *);AppLayerTxData *(*GetTxData)(void *tx);bool (*ApplyTxConfig)(void *state, void *tx, int mode, AppLayerTxConfig);void (*SetStreamDepthFlag)(void *tx, uint8_t flags);/* each app-layer has its own value */uint32_t stream_depth;/* Indicates the direction the parser is ready to see the data* the first time for a flow. Values accepted -* STREAM_TOSERVER, STREAM_TOCLIENT */uint8_t first_data_dir;/* Option flags such as supporting gaps or not. */uint32_t option_flags;/* coccinelle: AppLayerParserProtoCtx:option_flags:APP_LAYER_PARSER_OPT_ */uint32_t internal_flags;/* coccinelle: AppLayerParserProtoCtx:internal_flags:APP_LAYER_PARSER_INT_ */#ifdef UNITTESTSvoid (*RegisterUnittests)(void);#endif} AppLayerParserProtoCtx;http解析器的 HTTP state memorytypedef struct HtpState_ {/* Connection parser structure for each connection */htp_connp_t *connp;/* Connection structure for each connection */htp_conn_t *conn;Flow *f;/**< Needed to retrieve the original flow when using HTPLib callbacks */uint64_t transaction_cnt;uint64_t store_tx_id;FileContainer *files_ts;FileContainer *files_tc;const struct HTPCfgRec_ *cfg;uint16_t flags;uint16_t events;uint16_t htp_messages_offset; /**< offset into conn->messages list */uint32_t file_track_id; /**< used to assign file track ids to files */uint64_t last_request_data_stamp;uint64_t last_response_data_stamp;} HtpState;
SCHInfoLoadFromConfig：从配置文件中载入host os policy(主机OS策略)信息
radix tree保存所有主机策略信息
SigTableSetup：初始化检测引擎，主要是注册检测引擎所支持的规则格式（跟Snort规则基本一致）中的关键字，比如sid、priority、msg、within、distance等等
typedef struct SigTableElmt_ {/** Packet match function pointer */int (*Match)(DetectEngineThreadCtx *, Packet *, const Signature *, const SigMatchCtx *);/** AppLayer TX match function pointer */int (*AppLayerTxMatch)(DetectEngineThreadCtx *, Flow *,uint8_t flags, void *alstate, void *txv,const Signature *, const SigMatchCtx *);/** File match function pointer */int (*FileMatch)(DetectEngineThreadCtx *,Flow *, /**< *LOCKED* flow */uint8_t flags, File *, const Signature *, const SigMatchCtx *);/** InspectionBuffer transformation callback */void (*Transform)(InspectionBuffer *, void *context);bool (*TransformValidate)(const uint8_t *content, uint16_t content_len, void *context);/** keyword setup function pointer */int (*Setup)(DetectEngineCtx *, Signature *, const char *);bool (*SupportsPrefilter)(const Signature *s);int (*SetupPrefilter)(DetectEngineCtx *de_ctx, struct SigGroupHead_ *sgh);void (*Free)(DetectEngineCtx *, void *);#ifdef UNITTESTSvoid (*RegisterTests)(void);#endifuint16_t flags;/* coccinelle: SigTableElmt:flags:SIGMATCH_ *//** better keyword to replace the current one */uint16_t alternative;const char *name;/**< keyword name alias */const char *alias; /**< name alias */const char *desc;const char *url;} SigTableElmt;
规则结构体
/** \brief Signature container */typedef struct Signature_ {uint32_t flags;/* coccinelle: Signature:flags:SIG_FLAG_ */AppProto alproto;uint16_t dsize_low;uint16_t dsize_high;SignatureMask mask;SigIntId num; /**< signature number, internal id *//** inline -- action */uint8_t action;uint8_t file_flags;/** addresses, ports and proto this sig matches on */DetectProto proto;/** classification id **/uint16_t class_id;/** ipv4 match arrays */uint16_t addr_dst_match4_cnt;uint16_t addr_src_match4_cnt;uint16_t addr_dst_match6_cnt;uint16_t addr_src_match6_cnt;DetectMatchAddressIPv4 *addr_dst_match4;DetectMatchAddressIPv4 *addr_src_match4;/** ipv6 match arrays */DetectMatchAddressIPv6 *addr_dst_match6;DetectMatchAddressIPv6 *addr_src_match6;uint32_t id; /**< sid, set by the 'sid' rule keyword */uint32_t gid; /**< generator id */uint32_t rev;int prio;/** port settings for this signature */DetectPort *sp, *dp;#ifdef PROFILINGuint16_t profiling_id;#endif/** netblocks and hosts specified at the sid, in CIDR format */IPOnlyCIDRItem *CidrSrc, *CidrDst;DetectEngineAppInspectionEngine *app_inspect;DetectEnginePktInspectionEngine *pkt_inspect;/* Matching structures for the built-ins. The others are in* their inspect engines. */SigMatchData *sm_arrays[DETECT_SM_LIST_MAX];/* memory is still owned by the sm_lists/sm_arrays entry */const struct DetectFilestoreData_ *filestore_ctx;char *msg;/** classification message */char *class_msg;/** Reference */DetectReference *references;/** Metadata */DetectMetadataHead *metadata;char *sig_str;SignatureInitData *init_data;/** ptr to the next sig in the list */struct Signature_ *next;} Signature;
TmqhSetup：Tm-queue是各个模块（线程）之间传递数据的缓冲区
初始化queue handler（队列处理函数），这个是衔接线程模块和数据包队列之间的桥梁，目前共有5类handler：simple, nfq, packetpool, flow, ringbuffer。每类handler内部都有一个InHandler和OutHandler，一个用于从上一级队列中获取数据包，另一个用于处理完毕后将数据包送入下一级队列。 Packepool
通过read和write两个位置标记对packetpool(ringbuffer)这个循环队列进行进出操作。Simple
按照FIFO（先进先出）原则对缓冲区内容进行进出操作。Flow
出队的时候是按照FIFO进行，入队的时候对数据包的头部信息进行hash，然后将具有相同hash值的数据包放到一个缓冲区。
Tmqh tmqh_table[TMQH_SIZE]; RegisterAllModules：注册所有模块
里面注册了Suricata所支持的所有线程模块（Thread Module）。以pcap相关模块为例，TmModuleReceivePcapRegister函数注册了Pcap捕获模块，而TmModuleDecodePcapRegister函数注册了Pcap数据包解码模块。所谓注册，就是在tmm_modules模块数组中对应的那项中填充TmModule结构的所有字段，这些字段包括：模块名字、线程初始化函数、包处理或包获取函数、线程退出清理函数、一些标志位等等。
typedef struct TmModule_ {const char *name;/** thread handling */TmEcode (*ThreadInit)(ThreadVars *, const void *, void **);void (*ThreadExitPrintStats)(ThreadVars *, void *);TmEcode (*ThreadDeinit)(ThreadVars *, void *);/** the packet processing function */TmEcode (*Func)(ThreadVars *, Packet *, void *);TmEcode (*PktAcqLoop)(ThreadVars *, void *, void *);/** terminates the capture loop in PktAcqLoop */TmEcode (*PktAcqBreakLoop)(ThreadVars *, void *);TmEcode (*Management)(ThreadVars *, void *);/** global Init/DeInit */TmEcode (*Init)(void);TmEcode (*DeInit)(void);#ifdef UNITTESTSvoid (*RegisterTests)(void);#endifuint8_t cap_flags; /**< Flags to indicate the capability requierment ofthe given TmModule *//* Other flags used by the module */uint8_t flags;} TmModule;extern TmModule tmm_modules[TMM_SIZE]typedef enum {TMM_FLOWWORKER,TMM_DECODENFQ,TMM_VERDICTNFQ,TMM_RECEIVENFQ,TMM_RECEIVEPCAP,TMM_RECEIVEPCAPFILE,TMM_DECODEPCAP,TMM_DECODEPCAPFILE,TMM_RECEIVEPFRING,TMM_DECODEPFRING,TMM_RECEIVEPLUGIN,TMM_DECODEPLUGIN,TMM_RESPONDREJECT,TMM_DECODEIPFW,TMM_VERDICTIPFW,TMM_RECEIVEIPFW,TMM_RECEIVEERFFILE,TMM_DECODEERFFILE,TMM_RECEIVEERFDAG,TMM_DECODEERFDAG,TMM_RECEIVEAFP,TMM_DECODEAFP,TMM_RECEIVENETMAP,TMM_DECODENETMAP,TMM_ALERTPCAPINFO,TMM_RECEIVENAPATECH,TMM_DECODENAPATECH,TMM_STATSLOGGER,TMM_RECEIVENFLOG,TMM_DECODENFLOG,TMM_RECEIVEWINDIVERT,TMM_VERDICTWINDIVERT,TMM_DECODEWINDIVERT,TMM_FLOWMANAGER,TMM_FLOWRECYCLER,TMM_BYPASSEDFLOWMANAGER,TMM_DETECTLOADER,TMM_UNIXMANAGER,TMM_SIZE,} TmmId;
规则加载
LoadSignatures-》得到signutrue list
DetectEngineCtx_->sig_list + sig_cnt->排序ByAction，byFlowbits,，ByFlowvar，ByPktvar，ByHostbits，ByIPPairbit，ByPriority->runtime match structure （siggroup），siggroup的过程目前看是把之前的所有规则list通过ip，端口号和流分组到一个链表数组（sgh，并维护了一个 sigMask(规则有哪些匹配项)
$25 = {flags = 1572879, alproto = 0, dsize_low = 0, dsize_high = 0, mask = 0 '\000', num = 0, action = 1 '\001', file_flags = 0 '\000', proto = {proto = '\377' <repeats 32 times>, flags = 1 '\001'}, class = 26 '\032', addr_dst_match4_cnt = 1, addr_src_match4_cnt = 1, addr_dst_match6_cnt = 1, addr_src_match6_cnt = 1, addr_dst_match4 = 0x1867db0, addr_src_match4 = 0x1867df0, addr_dst_match6 = 0x1867f30, addr_src_match6 = 0x1867f00, id = 2260002, gid = 1, rev = 1, prio = 3, sp = 0x1867c70, dp = 0x1867ca0, CidrSrc = 0x1867bb0, CidrDst = 0x1867c40, app_inspect = 0x0, sm_arrays = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, filestore_ctx = 0x0, msg = 0x1867cd0 "SURICATA Applayer Detect protocol only one direction", class_msg = 0x1855880 "Generic Protocol Command Decode", references = 0x0, metadata = 0x0, sig_str = 0x1867980 "alert ip any any -> any any (msg:\"SURICATA Applayer Detect protocol only one direction\"; flow:established; app-layer-event:applayer_detect_protocol_only_one_direction; flowint:applayer.anomaly.count,+"..., init_data = 0x1856830, next = 0x0}(gdb) /* Signature flags *//** \note: additions should be added to the rule analyzer as well */#define SIG_FLAG_SRC_ANYBIT_U32(0) /**< source is any */#define SIG_FLAG_DST_ANYBIT_U32(1) /**< destination is any */#define SIG_FLAG_SP_ANY BIT_U32(2) /**< source port is any */#define SIG_FLAG_DP_ANY BIT_U32(3) /**< destination port is any */#define SIG_FLAG_NOALERTBIT_U32(4) /**< no alert flag is set */#define SIG_FLAG_DSIZE BIT_U32(5) /**< signature has a dsize setting */#define SIG_FLAG_APPLAYERBIT_U32(6) /**< signature applies to app layer instead of packets */#define SIG_FLAG_IPONLY BIT_U32(7) /**< ip only signature */
至此上电完毕，后面suricata则会根据run mode运行线程，循环抓包
