Posts

Showing posts from February, 2021

Hadoop Jars Link

 https://drive.google.com/file/d/1Fc8k625z5Y4KbG3VYUIAuNrsgwvubD8o/view?usp=sharing

Pig & Hive User Defined Functions Sample Code

Pig User Defined Function :  package com.hadoop;   import java.io.IOException;   import org.apache.pig.EvalFunc;   import org.apache.pig.data.Tuple;   public class TestUpper extends EvalFunc<String>   {       public String exec(Tuple input) throws IOException {             if (input == null || input.size() == 0)             return null;             try{                         String str = (String)input.get(0);             return str.toUpperCase();             }catch(Exception e){             throw new IOException("Caught exception processing input row ", e);                     }                 }   }   Hive User Defined Function: package com.lbrce.hiveudf; import org.apache.commons.lang.StringUtils; import org.apache.hadoop.hive.ql.exec.UDF; import org.apache.hadoop.io.Text; public class Strip extends UDF { private Text result = new Text();  public Text evaluate(Text str, String stripChars) {  if(str == null) {  return null;  }  result.set(StringUtils.strip(str.toStri

MapReduce Flights between Origin and Destination

MapReduce Flights between Origin and Destination: package com.lbrce.flight; import java.io.IOException; import java.util.Iterator; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.io.Text; import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat; import org.apache.hadoop.mapreduce.lib.input.TextInputFormat; import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.mapreduce.Mapper; import org.apache.hadoop.mapreduce.Reducer; import org.apache.hadoop.conf.Configuration; public class Flight { public static class FlightMapper extends Mapper<LongWritable, Text, Text, Text> { public void map(LongWritable arg0, Text Value, Context context) throws IOException, InterruptedException { String line = Value.toString(); if (!(line.length() == 0)) { String fno = line.substring(0, 4); String origin=line.sub