Real Time Face detection in Android using CameraX and MLKIT using JAVA

Pre-requisites:

• Firebase ML kit
• CameraX library
• Adding firebase to android app

Approach:

Step 1:

1. Create a new android project with compiled SDK version of 28.
2. The language used for this project is Java.

Step 2:

1. Login on Firebase.
2. Go to android studio Tools->Firebase->Firebase ML->click on use firebase ml to recognize image.
3. Refer this link to setup (https://firebase.google.com/docs/android/setup)
4. Dont forget to import services.json

2.Sync now

Step 4:Designing the UI.

I have used previewview to display camera feed.

<?xml version=”1.0" encoding=”utf-8"?>
<androidx.constraintlayout.widget.ConstraintLayout xmlns:android=”http://schemas.android.com/apk/res/android"
xmlns:app=”http://schemas.android.com/apk/res-auto"
xmlns:tools=”http://schemas.android.com/tools"
android:layout_width=”match_parent”
android:layout_height=”match_parent”
android:background=”#0C0B0B”
tools:context=”.MainActivity”>

<RelativeLayout
android:id=”@+id/conlay”
android:layout_width=”match_parent”
android:layout_height=”wrap_content”
app:layout_constraintEnd_toEndOf=”parent”
app:layout_constraintStart_toStartOf=”parent”
app:layout_constraintTop_toTopOf=”parent”>

<androidx.camera.view.PreviewView
android:id=”@+id/pv”
android:layout_width=”match_parent”
android:layout_height=”520dp”
app:layout_constraintBottom_toBottomOf=”parent”
app:layout_constraintEnd_toEndOf=”parent”
app:layout_constraintStart_toStartOf=”parent”>

</androidx.camera.view.PreviewView>

</RelativeLayout>

<com.google.android.material.floatingactionbutton.FloatingActionButton
android:id=”@+id/floatingActionButton2"
android:layout_width=”wrap_content”
android:layout_height=”wrap_content”
android:layout_marginStart=”177dp”
android:layout_marginEnd=”178dp”
android:layout_marginBottom=”16dp”
android:clickable=”true”
android:foregroundGravity=”center”
android:scaleType=”center”
app:backgroundTint=”#673AB7"
app:barrierMargin=”7dp”
app:fabCustomSize=”60dp”

app:layout_constraintBottom_toBottomOf=”parent”
app:layout_constraintEnd_toEndOf=”parent”
app:layout_constraintStart_toStartOf=”parent”
app:srcCompat=”@drawable/ic_baseline_mic_24" />

</androidx.constraintlayout.widget.ConstraintLayout>

Step 5: Render images to previewview using camerax .

private void renderimage() {

ListenableFuture<ProcessCameraProvider> val = ProcessCameraProvider.getInstance(this);

val.addListener(new Runnable() {
@RequiresApi(api = Build.VERSION_CODES.P)
@Override
public void run() {

try {

ProcessCameraProvider processCameraProvider= val.get();
preview = new Preview.Builder().setTargetResolution(new Size(900,1200)).build();

// ImageAnalysis imageAnalysis = new ImageAnalysis.Builder().build();
// imageAnalysis.setAnalyzer(getMainExecutor(),new BaseImage());

CameraSelector cameraSelector = new CameraSelector.Builder().requireLensFacing(CameraSelector.LENS_FACING_FRONT).build();

preview.setSurfaceProvider(previewView.getSurfaceProvider());

ImageCapture imageCapture = new ImageCapture.Builder().setCaptureMode(ImageCapture.CAPTURE_MODE_MINIMIZE_LATENCY).build();
ImageAnalysis imageAnalysis1 = new ImageAnalysis.Builder().setBackpressureStrategy(ImageAnalysis.STRATEGY_KEEP_ONLY_LATEST)
.setTargetResolution(new Size(900,1200)).build();
imageAnalysis1.setAnalyzer(ContextCompat.getMainExecutor(MainActivity.this)
,new BaseImage(MainActivity.this,textToSpeech,view,findViewById(R.id.textView3)));
processCameraProvider.unbindAll();

camera = processCameraProvider.bindToLifecycle(MainActivity.this,cameraSelector,preview,imageCapture,imageAnalysis1);
} catch (ExecutionException e) {
e.printStackTrace();
} catch (InterruptedException e) {
e.printStackTrace();
}

}
}, ContextCompat.getMainExecutor(MainActivity.this));

}

What this function will do :

1. It will pass images frame by frame and pass it to the ImageAnalysis.Analizer class.
2. Here cameraselector class is used to access camera whichever you want to use here i have used front camera.
3. ImageAnalysis is used to perform image processing on the image frame so you set your own analyer by extending ImageAnalysis.Analizer class .
4. Then all you need to do is bind the lifecycle of camera with the process.

This below code is the Analyzer class where you can detect all your Contours , landmarks,classify faces etc.

public class BaseImage implements ImageAnalysis.Analyzer {

FirebaseVisionFaceDetector firebaseVisionFaceDetector;

Context context;
TextToSpeech textToSpeech;
boolean speakornot = true;
boolean eyesopened = true;
boolean right = true;
boolean down = true;
boolean left = true;
boolean smile = true;
RelativeLayout rootview;
TextView textView;

public BaseImage(Context context,TextToSpeech textToSpeech,RelativeLayout view,TextView textView){

this.context = context;
this.textToSpeech = textToSpeech;
this.rootview = view;
this.textView = textView;
}

@SuppressLint(“UnsafeExperimentalUsageError”)
@Override
public void analyze(@NonNull ImageProxy image) {

Image n = image.getImage();
Log.d(“RAn analyzer — — — — — — — — — ->>>>>>>>>>” ,”Ran”);

FirebaseVisionFaceDetectorOptions a = new FirebaseVisionFaceDetectorOptions.Builder().setPerformanceMode(FirebaseVisionFaceDetectorOptions.FAST)
.setLandmarkMode(FirebaseVisionFaceDetectorOptions.ALL_LANDMARKS)
.setClassificationMode(FirebaseVisionFaceDetectorOptions.ALL_CLASSIFICATIONS)
.setMinFaceSize(0.2f)
.enableTracking()

.build();

int rotation = image.getImageInfo().getRotationDegrees();

FirebaseVisionImage p = FirebaseVisionImage.fromMediaImage(n, finddegree(rotation));

FirebaseVisionFaceDetector detector = FirebaseVision.getInstance().getVisionFaceDetector(a);

detector.detectInImage(p).addOnSuccessListener(new OnSuccessListener<List<FirebaseVisionFace>>() {
@Override
public void onSuccess(List<FirebaseVisionFace> firebaseVisionFaces) {

Log.d(“Got values — — — — — — — — — ->>>>>>>>>>” , firebaseVisionFaces.toString());

if(!firebaseVisionFaces.isEmpty()){

textView.setText(“Face detected”);

for(int i=0;i<firebaseVisionFaces.size();i++){
if(rootview.getChildCount()>1){
rootview.removeViewAt(1);
}

FirebaseVisionFace firebaseVisionFace = firebaseVisionFaces.get(i);

FirebaseVisionPoint firebaseVisionPoint = firebaseVisionFace.getLandmark(FirebaseVisionFaceLandmark.NOSE_BASE).getPosition();
FirebaseVisionPoint firebaseVisionPoint1 = firebaseVisionFace.getLandmark(FirebaseVisionFaceLandmark.MOUTH_BOTTOM).getPosition();

double x = Math.pow(firebaseVisionPoint1.getX() — firebaseVisionPoint.getX() , 2);
double y = Math.pow(firebaseVisionPoint1.getY() — firebaseVisionPoint.getY() , 2);

Log.d(“Head down figures — — — ->>>>” ,Math.sqrt(x + y) +” “+ ((0.125) * (firebaseVisionFace.getBoundingBox().width()*1.5)));

if(Math.sqrt(x + y) < ((0.111) * (firebaseVisionFace.getBoundingBox().width()*1.5))){

textToSpeech.speak(“Dont look down see straight”,TextToSpeech.QUEUE_FLUSH,null,null);
textView.setText(“Looking down”);

}

if(firebaseVisionFace.getHeadEulerAngleY() < -20){
textView.setText(“Left cheek”);

// left = false;

}

if(firebaseVisionFace.getHeadEulerAngleY() >20){
textView.setText(“Right cheek”);

// left = false;

}

if(firebaseVisionFace.getSmilingProbability()>0.8){

textView.setText(“Happy face”);
// textToSpeech.speak(“dont laugh too much”,TextToSpeech.QUEUE_FLUSH,null,null);

// smile = false;
}

View view1 = new Drawlayeraroundface(context,firebaseVisionFace.getBoundingBox());
rootview.addView(view1);
}

}else{
if(rootview.getChildCount()>1){
rootview.removeViewAt(1);
}

textView.setText(“Face Not detected”);

// Toast.makeText(context,”Not Detected”,Toast.LENGTH_LONG).show();

}

image.close();

}

}
).addOnFailureListener(new OnFailureListener() {
@Override
public void onFailure(@NonNull Exception e) {
Toast.makeText(context,”Not Detected”,Toast.LENGTH_LONG).show();
image.close();

}

});

}