Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
The table of contents is too big for display.
Diff view
Diff view
  •  
  •  
  •  
Empty file added .mvn/.gitkeep
Empty file.
11 changes: 11 additions & 0 deletions .mvn/jvm.config
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
--add-exports jdk.compiler/com.sun.tools.javac.code=ALL-UNNAMED
--add-exports jdk.compiler/com.sun.tools.javac.comp=ALL-UNNAMED
--add-exports jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED
--add-exports jdk.compiler/com.sun.tools.javac.main=ALL-UNNAMED
--add-exports jdk.compiler/com.sun.tools.javac.model=ALL-UNNAMED
--add-exports jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED
--add-exports jdk.compiler/com.sun.tools.javac.processing=ALL-UNNAMED
--add-exports jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED
--add-exports jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED
--add-opens jdk.compiler/com.sun.tools.javac.code=ALL-UNNAMED
--add-opens jdk.compiler/com.sun.tools.javac.comp=ALL-UNNAMED
10 changes: 10 additions & 0 deletions examples/storm-hdfs-examples/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -93,6 +93,16 @@
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.openrewrite.maven</groupId>
<artifactId>rewrite-maven-plugin</artifactId>
<!--Note - the version/config would be inherited-->
</plugin>
<plugin>
<groupId>org.codehaus.gmavenplus</groupId>
<artifactId>gmavenplus-plugin</artifactId>
<!--Note - the version/config would be inherited-->
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-checkstyle-plugin</artifactId>
Expand Down
Original file line number Diff line number Diff line change
@@ -1,12 +1,18 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
* 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
* Licensed to the Apache Software Foundation (ASF) under one or more contributor license
* agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership. The ASF licenses this file to
* you under the Apache License, Version
* 2.0 (the "License"); you may not use this file except in compliance with the License. You may
* obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
* <p>http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
* <p>Unless required by applicable law or agreed to in writing, software distributed under the
* License
* is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
* specific language governing permissions
* and limitations under the License.
*/

Expand Down Expand Up @@ -56,7 +62,8 @@ public static void main(String[] args) throws Exception {
SyncPolicy syncPolicy = new CountSyncPolicy(1000);

// rotate files when they reach 5MB
FileRotationPolicy rotationPolicy = new TimedRotationPolicy(1.0f, TimedRotationPolicy.TimeUnit.MINUTES);
FileRotationPolicy rotationPolicy = new TimedRotationPolicy(1.0f,
TimedRotationPolicy.TimeUnit.MINUTES);

FileNameFormat fileNameFormat = new DefaultFileNameFormat()
.withPath("/tmp/foo/")
Expand Down Expand Up @@ -91,7 +98,9 @@ public static void main(String[] args) throws Exception {
if (args.length == 3) {
topoName = args[2];
} else if (args.length > 3) {
System.out.println("Usage: HdfsFileTopology [hdfs url] [hdfs yaml config file] <topology name>");
System.out
.println("Usage: HdfsFileTopology [hdfs url] [hdfs yaml config file] "
+ "<topology name>");
return;
}
StormSubmitter.submitTopology(topoName, config, builder.createTopology());
Expand All @@ -101,7 +110,7 @@ public static void waitForSeconds(int seconds) {
try {
Thread.sleep(seconds * 1000);
} catch (InterruptedException e) {
//ignore
// ignore
}
}

Expand Down Expand Up @@ -145,7 +154,8 @@ public void nextTuple() {
total++;
if (count > 20000) {
count = 0;
System.out.println("Pending count: " + this.pending.size() + ", total: " + this.total);
System.out.println("Pending count: " + this.pending.size() + ", total: "
+ this.total);
}
Thread.yield();
}
Expand All @@ -168,7 +178,8 @@ public static class MyBolt extends BaseRichBolt {
private OutputCollector collector;

@Override
public void prepare(Map<String, Object> config, TopologyContext context, OutputCollector collector) {
public void prepare(Map<String, Object> config, TopologyContext context,
OutputCollector collector) {
this.counts = new HashMap<String, Long>();
this.collector = collector;
}
Expand Down
Original file line number Diff line number Diff line change
@@ -1,12 +1,18 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
* 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
* Licensed to the Apache Software Foundation (ASF) under one or more contributor license
* agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership. The ASF licenses this file to
* you under the Apache License, Version
* 2.0 (the "License"); you may not use this file except in compliance with the License. You may
* obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
* <p>http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
* <p>Unless required by applicable law or agreed to in writing, software distributed under the
* License
* is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
* specific language governing permissions
* and limitations under the License.
*/

Expand All @@ -25,8 +31,8 @@
import org.apache.storm.hdfs.bolt.format.DefaultSequenceFormat;
import org.apache.storm.hdfs.bolt.format.FileNameFormat;
import org.apache.storm.hdfs.bolt.rotation.FileRotationPolicy;
import org.apache.storm.hdfs.bolt.rotation.FileSizeRotationPolicy;
import org.apache.storm.hdfs.bolt.rotation.FileSizeRotationPolicy.Units;
import org.apache.storm.hdfs.bolt.rotation.FileSizeRotationPolicy;
import org.apache.storm.hdfs.bolt.sync.CountSyncPolicy;
import org.apache.storm.hdfs.bolt.sync.SyncPolicy;
import org.apache.storm.hdfs.common.rotation.MoveFileAction;
Expand Down Expand Up @@ -94,7 +100,9 @@ public static void main(String[] args) throws Exception {
if (args.length == 3) {
topoName = args[2];
} else if (args.length > 3) {
System.out.println("Usage: SequenceFileTopology [hdfs url] [hdfs yaml config file] <topology name>");
System.out
.println("Usage: SequenceFileTopology [hdfs url] [hdfs yaml config file] "
+ "<topology name>");
return;
}
StormSubmitter.submitTopology(topoName, config, builder.createTopology());
Expand All @@ -104,14 +112,12 @@ public static void waitForSeconds(int seconds) {
try {
Thread.sleep(seconds * 1000);
} catch (InterruptedException e) {
//ignore
// ignore
}
}


public static class SentenceSpout extends BaseRichSpout {


private ConcurrentHashMap<UUID, Values> pending;
private SpoutOutputCollector collector;
private String[] sentences = {
Expand Down Expand Up @@ -151,7 +157,8 @@ public void nextTuple() {
total++;
if (count > 20000) {
count = 0;
System.out.println("Pending count: " + this.pending.size() + ", total: " + this.total);
System.out.println("Pending count: " + this.pending.size() + ", total: "
+ this.total);
}
Thread.yield();
}
Expand All @@ -169,14 +176,14 @@ public void fail(Object msgId) {
}
}


public static class MyBolt extends BaseRichBolt {

private HashMap<String, Long> counts = null;
private OutputCollector collector;

@Override
public void prepare(Map<String, Object> config, TopologyContext context, OutputCollector collector) {
public void prepare(Map<String, Object> config, TopologyContext context,
OutputCollector collector) {
this.counts = new HashMap<String, Long>();
this.collector = collector;
}
Expand All @@ -186,7 +193,6 @@ public void execute(Tuple tuple) {
collector.ack(tuple);
}


@Override
public void declareOutputFields(OutputFieldsDeclarer declarer) {
// this bolt does not emit anything
Expand Down
Original file line number Diff line number Diff line change
@@ -1,12 +1,18 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
* 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
* Licensed to the Apache Software Foundation (ASF) under one or more contributor license
* agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership. The ASF licenses this file to
* you under the Apache License, Version
* 2.0 (the "License"); you may not use this file except in compliance with the License. You may
* obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
* <p>http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
* <p>Unless required by applicable law or agreed to in writing, software distributed under the
* License
* is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
* specific language governing permissions
* and limitations under the License.
*/

Expand Down Expand Up @@ -35,28 +41,36 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;


public class HdfsSpoutTopology {

public static final String SPOUT_ID = "hdfsspout";
public static final String BOLT_ID = "constbolt";

/**
* Copies text file content from sourceDir to destinationDir. Moves source files into sourceDir after its done consuming
* Copies text file content from sourceDir to destinationDir. Moves source files into sourceDir
* after its done consuming
*/
public static void main(String[] args) throws Exception {
// 0 - validate args
if (args.length < 7) {
System.err.println("Please check command line arguments.");
System.err.println("Usage :");
System.err.println(
HdfsSpoutTopology.class.toString() + " topologyName hdfsUri fileFormat sourceDir sourceArchiveDir badDir destinationDir.");
HdfsSpoutTopology.class.toString()
+ " topologyName hdfsUri fileFormat sourceDir sourceArchiveDir badDir "
+ "destinationDir.");
System.err.println(" topologyName - topology name.");
System.err.println(" hdfsUri - hdfs name node URI");
System.err.println(" fileFormat - Set to 'TEXT' for reading text files or 'SEQ' for sequence files.");
System.err
.println(" fileFormat - Set to 'TEXT' for reading text files or 'SEQ' for "
+ "sequence files.");
System.err.println(" sourceDir - read files from this HDFS dir using HdfsSpout.");
System.err.println(" archiveDir - after a file in sourceDir is read completely, it is moved to this HDFS location.");
System.err.println(" badDir - files that cannot be read properly will be moved to this HDFS location.");
System.err
.println(" archiveDir - after a file in sourceDir is read completely, it is "
+ "moved to this HDFS location.");
System.err
.println(" badDir - files that cannot be read properly will be moved to this "
+ "HDFS location.");
System.err.println(" spoutCount - Num of spout instances.");
System.err.println();
System.exit(-1);
Expand Down Expand Up @@ -153,7 +167,8 @@ public ConstBolt() {
}

@Override
public void prepare(Map<String, Object> conf, TopologyContext context, OutputCollector collector) {
public void prepare(Map<String, Object> conf, TopologyContext context,
OutputCollector collector) {
this.collector = collector;
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -41,11 +41,15 @@
public class TridentFileTopology {

public static StormTopology buildTopology(String hdfsUrl) {
FixedBatchSpout spout = new FixedBatchSpout(new Fields("sentence", "key"), 1000, new Values("the cow jumped over the moon", 1L),
new Values("the man went to the store and bought some candy", 2L),
new Values("four score and seven years ago", 3L),
FixedBatchSpout spout = new FixedBatchSpout(new Fields("sentence", "key"), 1000,
new Values("the cow jumped over the moon", 1L),
new Values("the man went to the store and "
+ "bought some candy", 2L),
new Values("four score and seven years ago",
3L),
new Values("how many apples can you eat", 4L),
new Values("to be or not to be the person", 5L));
new Values("to be or not to be the person",
5L));
spout.setCycle(true);

TridentTopology topology = new TridentTopology();
Expand All @@ -61,7 +65,8 @@ public static StormTopology buildTopology(String hdfsUrl) {
RecordFormat recordFormat = new DelimitedRecordFormat()
.withFields(hdfsFields);

FileRotationPolicy rotationPolicy = new FileSizeRotationPolicy(5.0f, FileSizeRotationPolicy.Units.MB);
FileRotationPolicy rotationPolicy = new FileSizeRotationPolicy(5.0f,
FileSizeRotationPolicy.Units.MB);

HdfsState.Options options = new HdfsState.HdfsFileOptions()
.withFileNameFormat(fileNameFormat)
Expand Down Expand Up @@ -91,7 +96,9 @@ public static void main(String[] args) throws Exception {
if (args.length == 3) {
topoName = args[2];
} else if (args.length > 3) {
System.out.println("Usage: TridentFileTopology [hdfs url] [hdfs yaml config file] <topology name>");
System.out
.println("Usage: TridentFileTopology [hdfs url] [hdfs yaml config file] "
+ "<topology name>");
return;
}
conf.setNumWorkers(3);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -41,11 +41,15 @@
public class TridentSequenceTopology {

public static StormTopology buildTopology(String hdfsUrl) {
FixedBatchSpout spout = new FixedBatchSpout(new Fields("sentence", "key"), 1000, new Values("the cow jumped over the moon", 1L),
new Values("the man went to the store and bought some candy", 2L),
new Values("four score and seven years ago", 3L),
FixedBatchSpout spout = new FixedBatchSpout(new Fields("sentence", "key"), 1000,
new Values("the cow jumped over the moon", 1L),
new Values("the man went to the store and "
+ "bought some candy", 2L),
new Values("four score and seven years ago",
3L),
new Values("how many apples can you eat", 4L),
new Values("to be or not to be the person", 5L));
new Values("to be or not to be the person",
5L));
spout.setCycle(true);

TridentTopology topology = new TridentTopology();
Expand All @@ -58,7 +62,8 @@ public static StormTopology buildTopology(String hdfsUrl) {
.withPrefix("trident")
.withExtension(".seq");

FileRotationPolicy rotationPolicy = new FileSizeRotationPolicy(5.0f, FileSizeRotationPolicy.Units.MB);
FileRotationPolicy rotationPolicy = new FileSizeRotationPolicy(5.0f,
FileSizeRotationPolicy.Units.MB);

HdfsState.Options seqOpts = new HdfsState.SequenceFileOptions()
.withFileNameFormat(fileNameFormat)
Expand Down Expand Up @@ -88,7 +93,8 @@ public static void main(String[] args) throws Exception {
if (args.length == 3) {
topoName = args[2];
} else if (args.length > 3) {
System.out.println("Usage: TridentSequenceTopology <hdfs_config_yaml> [<topology name>]");
System.out
.println("Usage: TridentSequenceTopology <hdfs_config_yaml> [<topology name>]");
return;
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,8 @@ public BoundedTupleSpout(long totalTuples, Fields outputFields, ValuesFactory va
}

@Override
public void open(Map<String, Object> conf, TopologyContext context, SpoutOutputCollector collector) {
public void open(Map<String, Object> conf, TopologyContext context,
SpoutOutputCollector collector) {
this.collector = collector;
// Each task walks its own stride of the sequence, so the spout can be parallelised without
// any two tasks emitting the same row.
Expand Down
10 changes: 10 additions & 0 deletions examples/storm-jdbc-examples/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -85,6 +85,16 @@
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.openrewrite.maven</groupId>
<artifactId>rewrite-maven-plugin</artifactId>
<!--Note - the version/config would be inherited-->
</plugin>
<plugin>
<groupId>org.codehaus.gmavenplus</groupId>
<artifactId>gmavenplus-plugin</artifactId>
<!--Note - the version/config would be inherited-->
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-checkstyle-plugin</artifactId>
Expand Down
Loading