corsaro_geodb.c File Reference

Corsaro CSV Database plugin. More...

Go to the source code of this file.

Macros
#define	CORSARO_GEODB_MAGIC   0x474F4442
	The magic number for this plugin - "GODB". More...
#define	PLUGIN_NAME   "geodb"
	The name of this plugin. More...
#define	MAXMIND_NAME   (corsaro_geo_get_provider_name(CORSARO_GEO_PROVIDER_MAXMIND))
	The name of the maxmind provider. More...
#define	NETACQ_EDGE_NAME   (corsaro_geo_get_provider_name(CORSARO_GEO_PROVIDER_NETACQ_EDGE))
	The name of the netacq edge provider. More...
#define	DEFAULT_PROVIDER_NAME   MAXMIND_NAME
	The default provider name. More...
#define	BUFFER_LEN   1024
	The length of the static line buffer. More...
#define	MAXMIND_HEADER_ROW_CNT   2
	The number of header rows in the maxmind CSV files. More...
#define	NETACQ_EDGE_HEADER_ROW_CNT   1
	The number of header rows in the netacq edge CSV files. More...
#define	LOCATIONS_FILE_NAME   "GeoLiteCity-Location.csv.gz"
	The default file name for the locations file. More...
#define	BLOCKS_FILE_NAME   "GeoLiteCity-Blocks.csv.gz"
	The default file name for the blocks file. More...
#define	STATE(corsaro)   (CORSARO_PLUGIN_STATE(corsaro, geodb, CORSARO_PLUGIN_ID_GEODB))
	Extends the generic plugin state convenience macro in corsaro_plugin.h. More...
#define	PLUGIN(corsaro)   (CORSARO_PLUGIN_PLUGIN(corsaro, CORSARO_PLUGIN_ID_GEODB))
	Extends the generic plugin plugin convenience macro in corsaro_plugin.h. More...

Typedefs
typedef enum maxmind_locations_cols	maxmind_locations_cols_t
	The columns in the maxmind locations CSV file. More...
typedef enum netacq_edge_locations_cols	netacq_edge_locations_cols_t
	The columns in the netacq_edge locations CSV file. More...
typedef enum blocks_cols	blocks_cols_t
	The columns in the maxmind locations CSV file. More...

Enumerations
enum	maxmind_locations_cols {   MAXMIND_LOCATION_COL_ID = 0, MAXMIND_LOCATION_COL_CC = 1, MAXMIND_LOCATION_COL_REGION = 2, MAXMIND_LOCATION_COL_CITY = 3,   MAXMIND_LOCATION_COL_POSTAL = 4, MAXMIND_LOCATION_COL_LAT = 5, MAXMIND_LOCATION_COL_LONG = 6, MAXMIND_LOCATION_COL_METRO = 7,   MAXMIND_LOCATION_COL_AREA = 8, MAXMIND_LOCATION_COL_COUNT = 9 }
	The columns in the maxmind locations CSV file. More...
enum	netacq_edge_locations_cols {   NETACQ_EDGE_LOCATION_COL_ID = 0, NETACQ_EDGE_LOCATION_COL_CC3 = 1, NETACQ_EDGE_LOCATION_COL_REGION = 2, NETACQ_EDGE_LOCATION_COL_CITY = 3,   NETACQ_EDGE_LOCATION_COL_CONN = 4, NETACQ_EDGE_LOCATION_COL_METRO = 5, NETACQ_EDGE_LOCATION_COL_LAT = 6, NETACQ_EDGE_LOCATION_COL_LONG = 7,   NETACQ_EDGE_LOCATION_COL_POSTAL = 8, NETACQ_EDGE_LOCATION_COL_CNTRYCODE = 9, NETACQ_EDGE_LOCATION_COL_RCODE = 10, NETACQ_EDGE_LOCATION_COL_CITYCODE = 11,   NETACQ_EDGE_LOCATION_COL_CONTCODE = 12, NETACQ_EDGE_LOCATION_COL_CC = 13, NETACQ_EDGE_LOCATION_COL_INTERNAL = 14, NETACQ_EDGE_LOCATION_COL_AREACODES = 15,   NETACQ_EDGE_LOCATION_COL_CNTRYCONF = 16, NETACQ_EDGE_LOCATION_COL_REGCONF = 17, NETACQ_EDGE_LOCATION_COL_CITYCONF = 18, NETACQ_EDGE_LOCATION_COL_POSTCONF = 19,   NETACQ_EDGE_LOCATION_COL_GMTOFF = 20, NETACQ_EDGE_LOCATION_COL_INDST = 21, NETACQ_EDGE_LOCATION_COL_TRAIL = 22, NETACQ_EDGE_LOCATION_COL_COUNT = 23 }
	The columns in the netacq_edge locations CSV file. More...
enum	blocks_cols { BLOCKS_COL_STARTIP = 0, BLOCKS_COL_ENDIP = 1, BLOCKS_COL_ID = 2, BLOCKS_COL_COUNT = 3 }
	The columns in the maxmind locations CSV file. More...

Functions
	KHASH_INIT (u16u16, uint16_t, uint16_t, 1, kh_int_hash_func, kh_int_hash_equal)
	Common plugin information across all instances. More...
static void	usage (corsaro_plugin_t *plugin)
	Print usage information to stderr. More...
static int	parse_args (corsaro_t *corsaro)
	Parse the arguments given to the plugin. More...
static void	parse_maxmind_location_cell (void *s, size_t i, void *data)
static void	parse_maxmind_location_row (int c, void *data)
	Handle an end-of-row event from the CSV parser. More...
static void	parse_netacq_edge_location_cell (void *s, size_t i, void *data)
	Parse a netacq location cell. More...
static void	parse_netacq_edge_location_row (int c, void *data)
	Handle an end-of-row event from the CSV parser. More...
static int	read_locations (corsaro_t *corsaro, corsaro_file_in_t *file)
	Read a locations file. More...
static void	parse_blocks_cell (void *s, size_t i, void *data)
	Parse a blocks cell. More...
static void	parse_blocks_row (int c, void *data)
static int	read_blocks (corsaro_t *corsaro, corsaro_file_in_t *file)
	Read a blocks file (maxmind or netacq) More...
static int	process_generic (corsaro_t *corsaro, corsaro_packet_state_t *state, uint32_t src_ip)
	Common code between process_packet and process_flowtuple. More...
corsaro_plugin_t *	corsaro_geodb_alloc (corsaro_t *corsaro)
	Implements the alloc function of the plugin API. More...
int	corsaro_geodb_probe_filename (const char *fname)
	Implements the probe_filename function of the plugin API. More...
int	corsaro_geodb_probe_magic (corsaro_in_t *corsaro, corsaro_file_in_t *file)
	Implements the probe_magic function of the plugin API. More...
int	corsaro_geodb_init_output (corsaro_t *corsaro)
	Implements the init_output function of the plugin API. More...
int	corsaro_geodb_init_input (corsaro_in_t *corsaro)
	Implements the init_input function of the plugin API. More...
int	corsaro_geodb_close_input (corsaro_in_t *corsaro)
	Implements the close_input function of the plugin API. More...
int	corsaro_geodb_close_output (corsaro_t *corsaro)
	Implements the close_output function of the plugin API. More...
off_t	corsaro_geodb_read_record (struct corsaro_in *corsaro, corsaro_in_record_type_t *record_type, corsaro_in_record_t *record)
	Implements the read_record function of the plugin API. More...
off_t	corsaro_geodb_read_global_data_record (corsaro_in_t *corsaro, corsaro_in_record_type_t *record_type, corsaro_in_record_t *record)
	Implements the read_global_data_record function of the plugin API. More...
int	corsaro_geodb_start_interval (corsaro_t *corsaro, corsaro_interval_t *int_start)
	Implements the start_interval function of the plugin API. More...
int	corsaro_geodb_end_interval (corsaro_t *corsaro, corsaro_interval_t *int_end)
	Implements the end_interval function of the plugin API. More...
int	corsaro_geodb_process_packet (corsaro_t *corsaro, corsaro_packet_t *packet)
	Implements the process_packet function of the plugin API. More...

Detailed Description

Corsaro CSV Database plugin.

This plugin is designed to tag packets based on the geolocation data found in the Maxmind Geo CSV format databases. That is, a database which consists of two tables: Blocks and Locations. See http://dev.maxmind.com/geoip/geolite for the free GeoLite versions of these databases.

It has been extended to understand the NetAcuity Edge database also, but only once it has been converted to this format. Contact corsa.nosp@m.ro-i.nosp@m.nfo@c.nosp@m.aida.nosp@m..org for more information about this feature.

Author

Alistair King

Definition in file corsaro_geodb.c.

Macro Definition Documentation

#define BLOCKS_FILE_NAME   "GeoLiteCity-Blocks.csv.gz"

The default file name for the blocks file.

Definition at line 193 of file corsaro_geodb.c.

Referenced by parse_args().

#define BUFFER_LEN   1024

The length of the static line buffer.

Definition at line 90 of file corsaro_geodb.c.

Referenced by read_blocks(), and read_locations().

#define CORSARO_GEODB_MAGIC   0x474F4442

The magic number for this plugin - "GODB".

Definition at line 73 of file corsaro_geodb.c.

Referenced by KHASH_INIT().

#define DEFAULT_PROVIDER_NAME   MAXMIND_NAME

The default provider name.

Definition at line 87 of file corsaro_geodb.c.

Referenced by usage().

#define LOCATIONS_FILE_NAME   "GeoLiteCity-Location.csv.gz"

The default file name for the locations file.

Definition at line 190 of file corsaro_geodb.c.

Referenced by parse_args().

#define MAXMIND_HEADER_ROW_CNT   2

The number of header rows in the maxmind CSV files.

Definition at line 118 of file corsaro_geodb.c.

Referenced by parse_blocks_cell(), and parse_maxmind_location_row().

#define MAXMIND_NAME   (corsaro_geo_get_provider_name(CORSARO_GEO_PROVIDER_MAXMIND))

The name of the maxmind provider.

Definition at line 79 of file corsaro_geodb.c.

Referenced by parse_args(), read_locations(), and usage().

#define NETACQ_EDGE_HEADER_ROW_CNT   1

The number of header rows in the netacq edge CSV files.

Definition at line 187 of file corsaro_geodb.c.

Referenced by parse_blocks_cell(), parse_netacq_edge_location_cell(), and parse_netacq_edge_location_row().

#define NETACQ_EDGE_NAME   (corsaro_geo_get_provider_name(CORSARO_GEO_PROVIDER_NETACQ_EDGE))

The name of the netacq edge provider.

Definition at line 83 of file corsaro_geodb.c.

Referenced by parse_args(), read_locations(), and usage().

#define PLUGIN

(

corsaro

)

(CORSARO_PLUGIN_PLUGIN(corsaro, CORSARO_PLUGIN_ID_GEODB))

Extends the generic plugin plugin convenience macro in corsaro_plugin.h.

Definition at line 239 of file corsaro_geodb.c.

Referenced by corsaro_geodb_close_output(), corsaro_geodb_init_output(), and parse_args().

#define PLUGIN_NAME   "geodb"

The name of this plugin.

Definition at line 76 of file corsaro_geodb.c.

Referenced by KHASH_INIT().

#define STATE

(

corsaro

)

(CORSARO_PLUGIN_STATE(corsaro, geodb, CORSARO_PLUGIN_ID_GEODB))

Extends the generic plugin state convenience macro in corsaro_plugin.h.

Definition at line 235 of file corsaro_geodb.c.

Referenced by corsaro_geodb_close_output(), parse_args(), parse_blocks_cell(), parse_maxmind_location_row(), parse_netacq_edge_location_cell(), parse_netacq_edge_location_row(), process_generic(), read_blocks(), and read_locations().

Typedef Documentation

typedef enum blocks_cols blocks_cols_t

The columns in the maxmind locations CSV file.

typedef enum maxmind_locations_cols maxmind_locations_cols_t

The columns in the maxmind locations CSV file.

typedef enum netacq_edge_locations_cols netacq_edge_locations_cols_t

The columns in the netacq_edge locations CSV file.

Enumeration Type Documentation

enum blocks_cols

The columns in the maxmind locations CSV file.

Enumerator
BLOCKS_COL_STARTIP	Range Start IP.
BLOCKS_COL_ENDIP	Range End IP.
BLOCKS_COL_ID	ID.
BLOCKS_COL_COUNT	Total number of columns in blocks table.

Definition at line 174 of file corsaro_geodb.c.

enum maxmind_locations_cols

The columns in the maxmind locations CSV file.

Enumerator
MAXMIND_LOCATION_COL_ID	ID.
MAXMIND_LOCATION_COL_CC	2 Char Country Code
MAXMIND_LOCATION_COL_REGION	Region String.
MAXMIND_LOCATION_COL_CITY	City String.
MAXMIND_LOCATION_COL_POSTAL	Postal Code String.
MAXMIND_LOCATION_COL_LAT	Latitude.
MAXMIND_LOCATION_COL_LONG	Longitude.
MAXMIND_LOCATION_COL_METRO	Metro Code.
MAXMIND_LOCATION_COL_AREA	Area Code (phone)
MAXMIND_LOCATION_COL_COUNT	Total number of columns in locations table.

Definition at line 93 of file corsaro_geodb.c.

enum netacq_edge_locations_cols

The columns in the netacq_edge locations CSV file.

Enumerator
NETACQ_EDGE_LOCATION_COL_ID	ID.
NETACQ_EDGE_LOCATION_COL_CC3	3 Char Country Code
NETACQ_EDGE_LOCATION_COL_REGION	Region String.
NETACQ_EDGE_LOCATION_COL_CITY	City String.
NETACQ_EDGE_LOCATION_COL_CONN	Connection Speed String.
NETACQ_EDGE_LOCATION_COL_METRO	Metro Code.
NETACQ_EDGE_LOCATION_COL_LAT	Latitude.
NETACQ_EDGE_LOCATION_COL_LONG	Longitude.
NETACQ_EDGE_LOCATION_COL_POSTAL	Postal Code.
NETACQ_EDGE_LOCATION_COL_CNTRYCODE	Country Code.
NETACQ_EDGE_LOCATION_COL_RCODE	Region Code.
NETACQ_EDGE_LOCATION_COL_CITYCODE	City Code.
NETACQ_EDGE_LOCATION_COL_CONTCODE	Continent Code.
NETACQ_EDGE_LOCATION_COL_CC	2 Char Country Code
NETACQ_EDGE_LOCATION_COL_INTERNAL	Internal Code.
NETACQ_EDGE_LOCATION_COL_AREACODES	Area Codes (plural)
NETACQ_EDGE_LOCATION_COL_CNTRYCONF	Country-Conf ??
NETACQ_EDGE_LOCATION_COL_REGCONF	Region-Conf ??
NETACQ_EDGE_LOCATION_COL_CITYCONF	City-Conf ??
NETACQ_EDGE_LOCATION_COL_POSTCONF	Postal-Conf ??
NETACQ_EDGE_LOCATION_COL_GMTOFF	GMT-Offset.
NETACQ_EDGE_LOCATION_COL_INDST	In CST.
NETACQ_EDGE_LOCATION_COL_TRAIL	Trailing comma.
NETACQ_EDGE_LOCATION_COL_COUNT	Total number of columns in the locations table.

Definition at line 121 of file corsaro_geodb.c.

Function Documentation

corsaro_plugin_t* corsaro_geodb_alloc

(

corsaro_t *

corsaro

)

Implements the alloc function of the plugin API.

Definition at line 1158 of file corsaro_geodb.c.

int corsaro_geodb_close_input

(

corsaro_in_t *

corsaro

)

Implements the close_input function of the plugin API.

Definition at line 1297 of file corsaro_geodb.c.

int corsaro_geodb_close_output

(

corsaro_t *

corsaro

)

Implements the close_output function of the plugin API.

Definition at line 1304 of file corsaro_geodb.c.

References corsaro_geo_free_provider(), corsaro_plugin_free_state(), PLUGIN, corsaro::plugin_manager, and STATE.

int corsaro_geodb_end_interval	(	corsaro_t *	corsaro,
		corsaro_interval_t *	int_end
	)

Implements the end_interval function of the plugin API.

Definition at line 1359 of file corsaro_geodb.c.

int corsaro_geodb_init_input

(

corsaro_in_t *

corsaro

)

Implements the init_input function of the plugin API.

Definition at line 1290 of file corsaro_geodb.c.

int corsaro_geodb_init_output

(

corsaro_t *

corsaro

)

Implements the init_output function of the plugin API.

Definition at line 1179 of file corsaro_geodb.c.

References corsaro_file_rclose(), corsaro_file_ropen(), CORSARO_GEO_DATASTRUCTURE_DEFAULT, corsaro_geo_get_maxmind_country_continent_list(), corsaro_geo_get_maxmind_iso2_list(), corsaro_geo_init_provider(), CORSARO_GEO_PROVIDER_DEFAULT_YES, corsaro_log(), corsaro_plugin_register_state(), malloc_zero(), parse_args(), PLUGIN, corsaro::plugin_manager, read_blocks(), read_locations(), and usage().

int corsaro_geodb_probe_filename

(

const char *

fname

)

Implements the probe_filename function of the plugin API.

Definition at line 1164 of file corsaro_geodb.c.

int corsaro_geodb_probe_magic	(	corsaro_in_t *	corsaro,
		corsaro_file_in_t *	file
	)

Implements the probe_magic function of the plugin API.

Definition at line 1171 of file corsaro_geodb.c.

int corsaro_geodb_process_packet	(	corsaro_t *	corsaro,
		corsaro_packet_t *	packet
	)

Implements the process_packet function of the plugin API.

Definition at line 1367 of file corsaro_geodb.c.

References LT_PKT, process_generic(), and corsaro_packet::state.

off_t corsaro_geodb_read_global_data_record	(	corsaro_in_t *	corsaro,
		corsaro_in_record_type_t *	record_type,
		corsaro_in_record_t *	record
	)

Implements the read_global_data_record function of the plugin API.

Definition at line 1342 of file corsaro_geodb.c.

off_t corsaro_geodb_read_record	(	struct corsaro_in *	corsaro,
		corsaro_in_record_type_t *	record_type,
		corsaro_in_record_t *	record
	)

Implements the read_record function of the plugin API.

Definition at line 1333 of file corsaro_geodb.c.

int corsaro_geodb_start_interval	(	corsaro_t *	corsaro,
		corsaro_interval_t *	int_start
	)

Implements the start_interval function of the plugin API.

Definition at line 1351 of file corsaro_geodb.c.

KHASH_INIT	(	u16u16	,
		uint16_t	,
		uint16_t	,
		1	,
		kh_int_hash_func	,
		kh_int_hash_equal
	)

Common plugin information across all instances.

Definition at line 195 of file corsaro_geodb.c.

References CORSARO_GEODB_MAGIC, CORSARO_PLUGIN_GENERATE_PTRS, CORSARO_PLUGIN_GENERATE_TAIL, CORSARO_PLUGIN_ID_GEODB, and PLUGIN_NAME.

static int parse_args ( corsaro_t *  corsaro )

static

Parse the arguments given to the plugin.

Todo:

add option to choose datastructure

Todo:

make this check for both .gz and non-.gz files

Definition at line 263 of file corsaro_geodb.c.

References corsaro_plugin::argc, corsaro_plugin::argv, BLOCKS_FILE_NAME, CORSARO_GEO_PROVIDER_MAXMIND, CORSARO_GEO_PROVIDER_NETACQ_EDGE, corsaro_log(), LOCATIONS_FILE_NAME, MAXMIND_NAME, NETACQ_EDGE_NAME, PLUGIN, STATE, and usage().

Referenced by corsaro_geodb_init_output().

static void parse_blocks_cell ( void *  s, size_t  i, void *  data  )

static

Parse a blocks cell.

Definition at line 889 of file corsaro_geodb.c.

References BLOCKS_COL_ENDIP, BLOCKS_COL_ID, BLOCKS_COL_STARTIP, CORSARO_GEO_PROVIDER_MAXMIND, CORSARO_GEO_PROVIDER_NETACQ_EDGE, corsaro_log(), MAXMIND_HEADER_ROW_CNT, NETACQ_EDGE_HEADER_ROW_CNT, and STATE.

Referenced by read_blocks().

static void parse_maxmind_location_row ( int  c, void *  data  )

static

Handle an end-of-row event from the CSV parser.

Definition at line 541 of file corsaro_geodb.c.

References corsaro_geo_init_record(), corsaro_log(), MAXMIND_HEADER_ROW_CNT, MAXMIND_LOCATION_COL_COUNT, record, and STATE.

Referenced by read_locations().

static void parse_netacq_edge_location_cell ( void *  s, size_t  i, void *  data  )

static

Parse a netacq location cell.

Definition at line 612 of file corsaro_geodb.c.

References corsaro_geo_record::city, corsaro_geo_record::conn_speed, corsaro_geo_record::continent_code, corsaro_log(), corsaro_geo_record::country_code, corsaro_geo_record::id, corsaro_geo_record::latitude, corsaro_geo_record::longitude, corsaro_geo_record::metro_code, NETACQ_EDGE_HEADER_ROW_CNT, NETACQ_EDGE_LOCATION_COL_AREACODES, NETACQ_EDGE_LOCATION_COL_CC, NETACQ_EDGE_LOCATION_COL_CC3, NETACQ_EDGE_LOCATION_COL_CITY, NETACQ_EDGE_LOCATION_COL_CITYCODE, NETACQ_EDGE_LOCATION_COL_CITYCONF, NETACQ_EDGE_LOCATION_COL_CNTRYCODE, NETACQ_EDGE_LOCATION_COL_CNTRYCONF, NETACQ_EDGE_LOCATION_COL_CONN, NETACQ_EDGE_LOCATION_COL_CONTCODE, NETACQ_EDGE_LOCATION_COL_GMTOFF, NETACQ_EDGE_LOCATION_COL_ID, NETACQ_EDGE_LOCATION_COL_INDST, NETACQ_EDGE_LOCATION_COL_INTERNAL, NETACQ_EDGE_LOCATION_COL_LAT, NETACQ_EDGE_LOCATION_COL_LONG, NETACQ_EDGE_LOCATION_COL_METRO, NETACQ_EDGE_LOCATION_COL_POSTAL, NETACQ_EDGE_LOCATION_COL_POSTCONF, NETACQ_EDGE_LOCATION_COL_RCODE, NETACQ_EDGE_LOCATION_COL_REGCONF, NETACQ_EDGE_LOCATION_COL_REGION, NETACQ_EDGE_LOCATION_COL_TRAIL, corsaro_geo_record::post_code, and STATE.

Referenced by read_locations().

static void parse_netacq_edge_location_row ( int  c, void *  data  )

static

Handle an end-of-row event from the CSV parser.

Definition at line 763 of file corsaro_geodb.c.

References corsaro_geo_init_record(), corsaro_log(), NETACQ_EDGE_HEADER_ROW_CNT, NETACQ_EDGE_LOCATION_COL_COUNT, record, and STATE.

Referenced by read_locations().

static int process_generic ( corsaro_t *  corsaro, corsaro_packet_state_t *  state, uint32_t  src_ip  )

static

Common code between process_packet and process_flowtuple.

Definition at line 1116 of file corsaro_geodb.c.

References corsaro_geo_dump_record(), corsaro_geo_get_default(), corsaro_geo_next_record(), corsaro_geo_provider_add_record(), corsaro_geo_provider_clear(), corsaro_geo_provider_lookup_record(), and STATE.

Referenced by corsaro_geodb_process_packet().

static int read_blocks ( corsaro_t *  corsaro, corsaro_file_in_t *  file  )

static

Read a blocks file (maxmind or netacq)

Definition at line 1061 of file corsaro_geodb.c.

References BUFFER_LEN, corsaro_file_rread(), corsaro_log(), parse_blocks_cell(), and STATE.

Referenced by corsaro_geodb_init_output().

static int read_locations ( corsaro_t *  corsaro, corsaro_file_in_t *  file  )

static

Read a locations file.

Definition at line 813 of file corsaro_geodb.c.

References BUFFER_LEN, corsaro_file_rread(), CORSARO_GEO_PROVIDER_MAXMIND, CORSARO_GEO_PROVIDER_NETACQ_EDGE, corsaro_log(), MAXMIND_NAME, NETACQ_EDGE_NAME, parse_maxmind_location_row(), parse_netacq_edge_location_cell(), parse_netacq_edge_location_row(), and STATE.

Referenced by corsaro_geodb_init_output().

static void usage ( corsaro_plugin_t *  plugin )

static

Print usage information to stderr.

Definition at line 243 of file corsaro_geodb.c.

References corsaro_plugin::argv, DEFAULT_PROVIDER_NAME, MAXMIND_NAME, and NETACQ_EDGE_NAME.

Referenced by corsaro_geodb_init_output(), and parse_args().