import javafx.embed.swt.FXCanvas;
import javafx.event.ActionEvent;
import javafx.event.EventHandler;
import javafx.geometry.Insets;
import javafx.scene.Parent;
import javafx.scene.Scene;
import javafx.scene.control.Button;
import javafx.scene.layout.StackPane;

import org.eclipse.jface.dialogs.Dialog;
import org.eclipse.swt.SWT;
import org.eclipse.swt.layout.FillLayout;
import org.eclipse.swt.widgets.Composite;
import org.eclipse.swt.widgets.Control;
import org.eclipse.swt.widgets.Shell;
import org.eclipse.swt.widgets.Display;

public class TestFx extends Dialog {
	public TestFx(Shell parent) {
		super(parent);
		setShellStyle(SWT.MODELESS | SWT.DIALOG_TRIM);
	}
	@Override
	public void okPressed() {
		TestFx dialog = new TestFx(Display.getDefault().getActiveShell());
		dialog.open();

	}
	@Override
	protected Control createDialogArea(Composite parent) {
		Composite container = (Composite) super.createDialogArea(parent);
		container.setLayout(new FillLayout());
		FXCanvas fxCanvas = new FXCanvas(container, SWT.NONE);
		Scene scene = new Scene(createScene());
		fxCanvas.setScene(scene);
		return container;
	}
	public Parent createScene() {
		StackPane pane = new StackPane();
		pane.setPadding(new Insets(10));
		Button button = new Button("Open");
		button.setOnAction(new EventHandler<ActionEvent>() {
			@Override public void handle(ActionEvent e) {
				TestFx dialog = new TestFx(Display.getDefault().getActiveShell());
				dialog.open();
			}
		});
		pane.getChildren().add(button);
		return pane;
	}		
}

projectMenu.add(new Action("TestFx"){
	@Override
	public void run() {
		TestFx dialog  = new TestFx(Display.getDefault().getActiveShell());
		dialog.open();
	}
});

public static void main(String[] args) {
	Display display = new Display();
	Shell shell = new Shell(display);
	TestFx dialog  = new TestFx(shell);
	dialog.open();
}

import javafx.embed.swt.FXCanvas;
import javafx.event.ActionEvent;
import javafx.event.EventHandler;
import javafx.geometry.Insets;
import javafx.scene.Scene;
import javafx.scene.control.Button;
import javafx.scene.layout.StackPane;

import org.eclipse.swt.SWT;
import org.eclipse.swt.events.MouseAdapter;
import org.eclipse.swt.events.MouseEvent;
import org.eclipse.swt.layout.FillLayout;
import org.eclipse.swt.widgets.Display;
import org.eclipse.swt.widgets.Shell;

public class TestFx  {
	static public void test() {
		Display display = Display.getDefault();
		
		Shell shell = new Shell(display.getActiveShell(),SWT.MODELESS | SWT.BORDER | SWT.CLOSE);
		shell.setLayout(new FillLayout());
		
		// fx open shell button
		FXCanvas fxCanvas = new FXCanvas(shell, SWT.NONE);
		StackPane pane = new StackPane();
		pane.setPadding(new Insets(10));
		Button button = new Button("FX Open Shell");
		button.setOnAction(new EventHandler<ActionEvent>() {
			@Override public void handle(ActionEvent e) {
				test();
			}
		});
		pane.getChildren().add(button);
		Scene scene = new Scene(pane);
		fxCanvas.setScene(scene);
		// swt open shell button
		org.eclipse.swt.widgets.Button b = new org.eclipse.swt.widgets.Button(shell, SWT.NONE);
		b.setText("SWT Open Shell");
		b.addMouseListener(new MouseAdapter() {
			@Override
			public void mouseDown(MouseEvent e) {
				test();
			}
		});
		shell.pack();
		shell.open ();
		/* event loop 
		 	- processes swt/fx events if test() was called from swt
			- processes only swt but no fx events if test() was called from fx
			  all fx events during this time will be processed only after the event loop terminates */
		while (!shell.isDisposed()) {
			try {
				if (!display.readAndDispatch()) {
					display.sleep();
				}
			} catch (Throwable e) {
				e.printStackTrace();
			}
		}		
	}
	// lockup does not occur if this is run as a standalone swt app
	public static void main(String[] args) {
		test();
	}
}

...
PlatformImpl$4.run() line: 176	
OS.DispatchMessageW(MSG) line: not available [native method]	
OS.DispatchMessage(MSG) line: 2546	
Display.readAndDispatch() line: 3756	
...

...
PlatformImpl$4.run() line: 176	
InvokeLaterDispatcher$Future.run() line: 76
OS.DispatchMessageW(MSG) line: not available [native method]	
OS.DispatchMessage(MSG) line: 2546	
Display.readAndDispatch() line: 3756	
...