.\bin\spark-submit examples\src\main\python\sql\streaming\structured_network_wordcount.py localhost 9999

添加文件

hadoop fs -appendToFile /tmp/country.csv /my/country

创建文件夹

hadoop fs -mkdir -p /my/files

删除文件夹

hadoop fs -rm -r -f /my
hadoop fs -rm -r -f /my/files

python -m pip install pandas -i https://mirrors.volces.com/pypi/simple/ --trusted-host mirrors.volces.com

1、下载地址
https://td-agent-package-browser.herokuapp.com/5/windows
选择LTS版本

2、配置路径
D:\fluentd\fluent\etc\fluent\fluentd.conf

3、运行

D:\fluentd\fluent\fluentd

- 阅读剩余部分 -

1、在hadoop on yarn环境基础上, 增加spark配置.
spark-env.sh

HADOOP_CONF_DIR=/usr/local/hadoop34/etc/hadoop
YARN_CONF_DIR=/usr/local/hadoop34/etc/hadoop

workers文件:

slave1

2、运行测试

./bin/spark-submit --master yarn --class org.apache.spark.examples.SparkPi ./examples/jars/spark-examples_2.12-3.5.5.jar 10

# 运行pyspark
./bin/pyspark --master yarn