Apache Flume:1.9.0
Apache Kafka:2.0.0
参考：https://github.com/apache/flume/blob/trunk/flume-ng-doc/sphinx/FlumeUserGuide.rst
1、flume配置文件
#agent1表示代理名称
agent1.sources=source1
agent1.sinks=sink1
agent1.channels=channel1
#配置source1
agent1.sources.source1.type=TAILDIR
agent1.sources.source1.filegroups = f1
agent1.sources.source1.filegroups.f1 = /data/log/tracy/.*log.*
agent1.sources.source1.channels=channel1
agent1.sources.source1.fileHeader = false
#加拦截器
agent1.sources.source1.interceptors = i1
#时间戳拦截器
agent1.sources.source1.interceptors.i1.type = timestamp
#配置channel1
agent1.channels.channel1.type=file
agent1.channels.channel1.checkpointDir=/data/flume/tracy/cheackpointDir
agent1.channels.channel1.dataDirs=/data/flume/tracy/dataDirs
#配置sink1
agent1.sinks.sink1.channel = channel1
agent1.sinks.sink1.type = org.apache.flume.sink.kafka.KafkaSink
agent1.sinks.sink1.kafka.topic = action.log
agent1.sinks.sink1.kafka.bootstrap.servers = bigdata-dev1:9092
agent1.sinks.sink1.kafka.flumeBatchSize = 20
agent1.sinks.sink1.kafka.producer.acks = 1
agent1.sinks.sink1.kafka.producer.linger.ms = 1
agent1.sinks.sink1.kafka.producer.compression.type = snappy
2、启动flume ng
./flume-ng agent -n agent1 -c ../conf -f ../conf/log2kafka.properties -Dflume.root.logger=DEBUG,console
3、kafka配置了auto.create.topics.enable true，所以不需要创建topic，如果关掉自动创建，需创建topic
4、启动kafka consumer: ./kafka-console-consumer.sh --topic action.log --from-beginning --bootstrap-server bigdata-dev1:9092
5、起一个造数据的脚本
[email protected]:/data/log/tracy# cat /opt/postgres/action_log.bash
#!/bin/bash
while true
do
m_t=`date "+%Y-%m-%d %H:%M:%S"`
UUID=$(cat /proc/sys/kernel/random/uuid)
ower_array=("Aaron" "Abbott" "Abel" "Abner" "Abraham" "Adair" "Adam" "Adolph" "Adonis" "Adrian" "Ahern" "Alan" "Albert" "Aldrich" "Alexander" "Alfred" "Alger" "Algernon" "Allen" "Alston" "Alva" "Alvis" "Amos" "Andre" "Andrew" "Andy" "Angelo" "Augus" "Ansel" "Antony" "Antoine" "Antonio" "Archer" "Archibald" "Aries" "Arlen" "Armand" "Armstrong" "Arno" "Arnold" "Arthur" "Arvin" "Asa" "Ashbur" "Atwood" "Aubrey" "August" "Augustine" "Avery" "Baird" "Baldwin" "Bancroft" "Bard" "Barlow" "Barnett" "Baron" "Barret" "Barry" "Bartholomew" "Bart" "Barton" "Bartley" "Basil" "Beacher" "Beau" "Beck" "Ben" "Benedict" "Benjamin" "Bennett" "Benson" "Berg" "Berger" "Bernard" "Bernie" "Bert" "Berton" "Bertram" "Bevis" "Bill" "Bing" "Bishop" "Blair" "Blake" "BliTHE" "Bob" "Booth" "Borg" "Boris" "Bowen" "Boyce" "Boyd" "Bradley" "Brady" "Brandon" "Brian" "Broderick" "Brook" "Bruce" "Bruno" "Buck" "Burgess" "Burke" "Burnell" "Burton" "Byron" "Caesar" "Calvin" "Carey" "Carl" "Carr" "Carter" "Cash" "Cecil" "Cedric" "Chad" "Channing" "Chapman" "Charles" "Chasel" "Chester" "Christian" "Christopher" "Clare" "Clarence" "Clark" "Claude" "Clement" "Cleveland" "Cliff" "Clifford" "Clyde" "Colbert" "Colby" "Colin" "Conrad" "Corey" "Cornelius" "Craig" "Curitis" "Cyril" "Dana" "Daniel" "Darcy" "Darnell" "Darren" "Dave" "David" "Dean" "Dempsey" "Dennis" "Derrick" "Devin" "Dick" "Dominic" "Don" "Donahue" "Donald" "Drew" "Duke" "Duncan" "Dunn" "Dwight" "Dylan" "Earl" "Ed" "Eden" "Edgar" "Edmund" "Edison" "Edward" "Edwiin" "Egbert" "Eli" "Elijah" "Elliot" "Ellis" "Elmer" "Elroy" "Elton" "Elvis" "Emmanuel" "Enoch" "Eric" "Ernest" "Eugene" "Evan" "Fabian" "Felix" "Fitzgerald" "Ford" "Frank" "Franklin" "Frederic" "Gale" "Gary" "Gavin" "Gene" "Geoffrey" "Geoff" "George" "Gerald" "Giles" "Glenn" "Gordon" "GREg" "Griffith" "Grover" "Guy" "Hale" "Haley" "Hamiltion" "Hardy" "Harlan" "Harley" "Harold" "Harry" "Harvey" "Hayden" "Henry" "Herman" "Hilary" "Hiram" "Hobart" "Hogan" "Horace" "Howar" "Hubery" "Hugh" "Hugo" "Humphrey" "Hunter" "Hyman" "Ian" "Ingram" "Ira" "Isaac" "Ivan" "Ives" "Jack" "Jacob" "James" "Jared" "Jason" "Jay" "Jeff" "Jeremy" "Jerome" "Jerry" "Jesse" "Jim" "Jo" "John" "Jonas" "Jonathan" "Joseph" "Joshua" "Joyce" "Julian" "Julius" "Justin" "Keith" "Kelly" "Ken" "Kent" "Kerr" "Kerwin" "Kim" "King" "Kirk" "Kyle" "Lance" "Larry" "Lawrence" "Leif" "Len" "Lennon" "Leo" "Les" "Lester" "Levi" "Lewis" "Lionel" "Lou" "Louis" "Lucien" "LuTHEr" "Lyle" "Lyndon" "Lynn" "Magee" "Mandel" "Marcus" "Marico" "Marlon" "Marsh" "Marshall" "Martin" "Marvin" "Matt" "Max" "Maximilian" "Meredith" "Merle" "Merlin" "Mick" "Mike" "Miles" "Milo" "Monroe" "Montague" "Moore" "Morgan" "Mortimer" "Morton" "Moses" "Murphy" "Murray" "Myron" "Nat" "Nathan" "Nathaniel" "Neil" "Nelson" "Newman" "Nicholas" "Nick" "Nigel" "Noah" "Noel" "Norman" "Norton" "Ogden" "Oliver" "Omar" "Osborn" "Oscar" "Osmond" "Oswald" "Otis" "Otto" "Owen" "Page" "Parker" "Paddy" "Paul" "Payne" "Perry" "Pete" "Peter" "Phil" "Philip" "Porter" "Prescott" "Primo" "Quincy" "Quinn" "Quintion" "Rachel" "Ralap" "Randolph" "Reg" "Regan" "Reginald" "Reuben" "Rex" "Robert" "Robin" "Rock" "Rod" "Roderick" "Rodney" "Ron" "Ronald" "Rory" "Roy" "Rudolf" "Rupert" "Ryan" "Sam" "Samuel" "Sandy" "Saxon" "Scott" "Sean" "Sebastian" "Sid" "Sidney" "Silvester" "Simon" "Stan" "Stanford" "StevenStevSteward")
ower_id=$[RANDOM%393+1]
org_id=$[ower_id%10+1]
echo $UUID,$m_t,${ower_array[$ower_id]} >> /data/log/tracy/2.log
sleep 5
done
6、kafka端开始有数据输出