C로 HDFS 다루기

2016/05/19

Makefile

CC = gcc -O3 -Wall -g -I /app/hdfs/include -L /app/hdfs/lib/native -L /app/jdk1.7.0_45/jre/lib/amd64/server

RECO_SRC = $(wildcard *.c)
DEBUG = -DDEBUG -DDEBUG_INFO -DDEBUG_DEBUG -Wall
LIBS = -lm -lpthread -lhdfs -ljvm

all: $(RECO_SRC)
                $(CC) -o hdfstest  $(RECO_SRC) $(DEBUG) $(LIBS)
.SUFFIXES: .c.o
        .c.o:
                $(CC) -c -o $@ $< $(DEBUG) $(LIBS)

clean :
        find ./ -name "*.*o" -exec rm -f {} +
        rm -f hdfstest

.

.

.

run.sh

#!/bin/sh
export CLASSPATH=`hadoop classpath --glob`
export LD_LIBRARY_PATH=/app/hdfs/lib/native:/app/jdk1.7/jre/lib/amd64/server

./hdfstest

.

.

.

hdfstest.c

#include <stdio.h>
#include <stdlib.h>

#include "hdfs.h"

int main()
{

    hdfsFS fs;
    hdfsFileInfo *fi;
    hdfsFile rfile;
    int i;
    int fnum;
    int rsize;
    char temp[4097];

    printf("hello hdfs\n");

    fs = hdfsConnect("default", 0);
    fi = hdfsListDirectory(fs, "/app" , &fnum);

    for (i = 0; i < fnum; i++) {

        printf("%02d:%s, %ld\n", i, (fi + i)->mName, (fi + i)->mSize);
//        rfile = hdfsOpenFile(fs, (fi + i)->mName, O_RDONLY, 0, 0, 0);
//        tsize += (fi + i)->mSize;
//        hdfsCloseFile(fs, rfile);

    }

    rfile = hdfsOpenFile(fs, "reco_mahout_input_item/000000_0", O_RDONLY, 0, 0, 0);

    rsize = 4096;

    rsize = hdfsRead(fs, rfile, (void *)temp, rsize);

    hdfsCloseFile(fs, rfile);
    temp[4096] = 0;

    printf("buf = %s", temp);

    hdfsDisconnect(fs);

    exit(0);

}

제로카피를 쓰자!

https://github.com/dawnsea/libhdfs_example

[t:/] is not "technology - root". dawnsea, rss